diff --git a/.gitignore b/.gitignore index 2dc8f56e65..f98ece8f4c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .python-version .env .venv +*.egg-info # OSX .DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5bfb263ae0..9885b3b51b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -228,7 +228,11 @@ export CORTEX_DEV_DEFAULT_PREDICTOR_IMAGE_REGISTRY="cortexlabs" export CORTEX_TELEMETRY_SENTRY_DSN="https://c334df915c014ffa93f2076769e5b334@sentry.io/1848098" export CORTEX_TELEMETRY_SEGMENT_WRITE_KEY="0WvoJyCey9z1W2EW7rYTPJUMRYat46dl" -alias cortex='$HOME/bin/cortex' # your path may be different depending on where you cloned the repo +# instruct the Python client to use your development CLI binary (update the path to point to your cortex repo) +export CORTEX_CLI_PATH="/bin/cortex" + +# create a cortex alias which runs your development CLI +alias cortex="$CORTEX_CLI_PATH" ``` Refresh your bash profile: diff --git a/test/README.md b/test/README.md index 1eb711f57d..680a154449 100644 --- a/test/README.md +++ b/test/README.md @@ -1,67 +1,5 @@ -# Examples +# Cortex Tests -## TensorFlow - -- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers. - -- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text. - -- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis. - -- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images. - -- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images. - -- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time. - -- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API. - -## Keras - -- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise. - -## PyTorch - -- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers. - -- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text. - -- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis. - -- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms. - -- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions. - -- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text. - -- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension. - -- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages. - -- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API. - -- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images. - -- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images. - -- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images. - -- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer. - -## ONNX - -- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers. - -- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset. - -- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API. - -## scikit-learn - -- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers. - -- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG. - -## spacy - -- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition. +- [Example APIs](apis) +- [End-to-end Tests](e2e) +- [Testing Utilities](utils) diff --git a/test/apis/README.md b/test/apis/README.md new file mode 100644 index 0000000000..1eb711f57d --- /dev/null +++ b/test/apis/README.md @@ -0,0 +1,67 @@ +# Examples + +## TensorFlow + +- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers. + +- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text. + +- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis. + +- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images. + +- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images. + +- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time. + +- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API. + +## Keras + +- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise. + +## PyTorch + +- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers. + +- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text. + +- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis. + +- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms. + +- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions. + +- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text. + +- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension. + +- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages. + +- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API. + +- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images. + +- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images. + +- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images. + +- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer. + +## ONNX + +- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers. + +- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset. + +- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API. + +## scikit-learn + +- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers. + +- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG. + +## spacy + +- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition. diff --git a/test/batch/image-classifier/README.md b/test/apis/batch/image-classifier/README.md similarity index 100% rename from test/batch/image-classifier/README.md rename to test/apis/batch/image-classifier/README.md diff --git a/test/batch/image-classifier/cortex.yaml b/test/apis/batch/image-classifier/cortex.yaml similarity index 100% rename from test/batch/image-classifier/cortex.yaml rename to test/apis/batch/image-classifier/cortex.yaml diff --git a/test/batch/image-classifier/predictor.py b/test/apis/batch/image-classifier/predictor.py similarity index 100% rename from test/batch/image-classifier/predictor.py rename to test/apis/batch/image-classifier/predictor.py diff --git a/test/batch/image-classifier/requirements.txt b/test/apis/batch/image-classifier/requirements.txt similarity index 100% rename from test/batch/image-classifier/requirements.txt rename to test/apis/batch/image-classifier/requirements.txt diff --git a/test/batch/image-classifier/sample.json b/test/apis/batch/image-classifier/sample.json similarity index 100% rename from test/batch/image-classifier/sample.json rename to test/apis/batch/image-classifier/sample.json diff --git a/test/batch/onnx/cortex.yaml b/test/apis/batch/onnx/cortex.yaml similarity index 100% rename from test/batch/onnx/cortex.yaml rename to test/apis/batch/onnx/cortex.yaml diff --git a/test/batch/onnx/predictor.py b/test/apis/batch/onnx/predictor.py similarity index 100% rename from test/batch/onnx/predictor.py rename to test/apis/batch/onnx/predictor.py diff --git a/test/batch/onnx/requirements.txt b/test/apis/batch/onnx/requirements.txt similarity index 100% rename from test/batch/onnx/requirements.txt rename to test/apis/batch/onnx/requirements.txt diff --git a/test/apis/batch/onnx/sample.json b/test/apis/batch/onnx/sample.json new file mode 100644 index 0000000000..eb45c463fd --- /dev/null +++ b/test/apis/batch/onnx/sample.json @@ -0,0 +1,3 @@ +[ + "https://i.imgur.com/PzXprwl.jpg" +] diff --git a/test/batch/tensorflow/cortex.yaml b/test/apis/batch/tensorflow/cortex.yaml similarity index 100% rename from test/batch/tensorflow/cortex.yaml rename to test/apis/batch/tensorflow/cortex.yaml diff --git a/test/batch/tensorflow/predictor.py b/test/apis/batch/tensorflow/predictor.py similarity index 100% rename from test/batch/tensorflow/predictor.py rename to test/apis/batch/tensorflow/predictor.py diff --git a/test/batch/tensorflow/requirements.txt b/test/apis/batch/tensorflow/requirements.txt similarity index 100% rename from test/batch/tensorflow/requirements.txt rename to test/apis/batch/tensorflow/requirements.txt diff --git a/test/apis/batch/tensorflow/sample.json b/test/apis/batch/tensorflow/sample.json new file mode 100644 index 0000000000..eb45c463fd --- /dev/null +++ b/test/apis/batch/tensorflow/sample.json @@ -0,0 +1,3 @@ +[ + "https://i.imgur.com/PzXprwl.jpg" +] diff --git a/test/keras/document-denoiser/README.md b/test/apis/keras/document-denoiser/README.md similarity index 100% rename from test/keras/document-denoiser/README.md rename to test/apis/keras/document-denoiser/README.md diff --git a/test/keras/document-denoiser/cortex.yaml b/test/apis/keras/document-denoiser/cortex.yaml similarity index 100% rename from test/keras/document-denoiser/cortex.yaml rename to test/apis/keras/document-denoiser/cortex.yaml diff --git a/test/keras/document-denoiser/predictor.py b/test/apis/keras/document-denoiser/predictor.py similarity index 100% rename from test/keras/document-denoiser/predictor.py rename to test/apis/keras/document-denoiser/predictor.py diff --git a/test/keras/document-denoiser/requirements.txt b/test/apis/keras/document-denoiser/requirements.txt similarity index 100% rename from test/keras/document-denoiser/requirements.txt rename to test/apis/keras/document-denoiser/requirements.txt diff --git a/test/keras/document-denoiser/sample.json b/test/apis/keras/document-denoiser/sample.json similarity index 100% rename from test/keras/document-denoiser/sample.json rename to test/apis/keras/document-denoiser/sample.json diff --git a/test/keras/document-denoiser/trainer.ipynb b/test/apis/keras/document-denoiser/trainer.ipynb similarity index 100% rename from test/keras/document-denoiser/trainer.ipynb rename to test/apis/keras/document-denoiser/trainer.ipynb diff --git a/test/live-reloading/onnx/README.md b/test/apis/live-reloading/onnx/README.md similarity index 100% rename from test/live-reloading/onnx/README.md rename to test/apis/live-reloading/onnx/README.md diff --git a/test/live-reloading/python/mpg-estimator/cortex.yaml b/test/apis/live-reloading/python/mpg-estimator/cortex.yaml similarity index 100% rename from test/live-reloading/python/mpg-estimator/cortex.yaml rename to test/apis/live-reloading/python/mpg-estimator/cortex.yaml diff --git a/test/live-reloading/python/mpg-estimator/predictor.py b/test/apis/live-reloading/python/mpg-estimator/predictor.py similarity index 97% rename from test/live-reloading/python/mpg-estimator/predictor.py rename to test/apis/live-reloading/python/mpg-estimator/predictor.py index eb643efcd7..122fac51fc 100644 --- a/test/live-reloading/python/mpg-estimator/predictor.py +++ b/test/apis/live-reloading/python/mpg-estimator/predictor.py @@ -1,5 +1,4 @@ import mlflow.sklearn -import numpy as np class PythonPredictor: diff --git a/test/live-reloading/python/mpg-estimator/requirements.txt b/test/apis/live-reloading/python/mpg-estimator/requirements.txt similarity index 100% rename from test/live-reloading/python/mpg-estimator/requirements.txt rename to test/apis/live-reloading/python/mpg-estimator/requirements.txt diff --git a/test/live-reloading/python/mpg-estimator/sample.json b/test/apis/live-reloading/python/mpg-estimator/sample.json similarity index 100% rename from test/live-reloading/python/mpg-estimator/sample.json rename to test/apis/live-reloading/python/mpg-estimator/sample.json diff --git a/test/live-reloading/tensorflow/README.md b/test/apis/live-reloading/tensorflow/README.md similarity index 100% rename from test/live-reloading/tensorflow/README.md rename to test/apis/live-reloading/tensorflow/README.md diff --git a/test/model-caching/onnx/multi-model-classifier/README.md b/test/apis/model-caching/onnx/multi-model-classifier/README.md similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/README.md rename to test/apis/model-caching/onnx/multi-model-classifier/README.md diff --git a/test/model-caching/onnx/multi-model-classifier/cortex.yaml b/test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/cortex.yaml rename to test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml diff --git a/test/model-caching/onnx/multi-model-classifier/predictor.py b/test/apis/model-caching/onnx/multi-model-classifier/predictor.py similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/predictor.py rename to test/apis/model-caching/onnx/multi-model-classifier/predictor.py diff --git a/test/model-caching/onnx/multi-model-classifier/requirements.txt b/test/apis/model-caching/onnx/multi-model-classifier/requirements.txt similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/requirements.txt rename to test/apis/model-caching/onnx/multi-model-classifier/requirements.txt diff --git a/test/model-caching/onnx/multi-model-classifier/sample.json b/test/apis/model-caching/onnx/multi-model-classifier/sample.json similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/sample.json rename to test/apis/model-caching/onnx/multi-model-classifier/sample.json diff --git a/test/model-caching/python/mpg-estimator/README.md b/test/apis/model-caching/python/mpg-estimator/README.md similarity index 100% rename from test/model-caching/python/mpg-estimator/README.md rename to test/apis/model-caching/python/mpg-estimator/README.md diff --git a/test/model-caching/python/mpg-estimator/cortex.yaml b/test/apis/model-caching/python/mpg-estimator/cortex.yaml similarity index 100% rename from test/model-caching/python/mpg-estimator/cortex.yaml rename to test/apis/model-caching/python/mpg-estimator/cortex.yaml diff --git a/test/model-caching/python/mpg-estimator/predictor.py b/test/apis/model-caching/python/mpg-estimator/predictor.py similarity index 97% rename from test/model-caching/python/mpg-estimator/predictor.py rename to test/apis/model-caching/python/mpg-estimator/predictor.py index 8fcdb50a5d..146bab6c90 100644 --- a/test/model-caching/python/mpg-estimator/predictor.py +++ b/test/apis/model-caching/python/mpg-estimator/predictor.py @@ -1,5 +1,4 @@ import mlflow.sklearn -import numpy as np class PythonPredictor: diff --git a/test/model-caching/python/mpg-estimator/requirements.txt b/test/apis/model-caching/python/mpg-estimator/requirements.txt similarity index 100% rename from test/model-caching/python/mpg-estimator/requirements.txt rename to test/apis/model-caching/python/mpg-estimator/requirements.txt diff --git a/test/model-caching/python/mpg-estimator/sample.json b/test/apis/model-caching/python/mpg-estimator/sample.json similarity index 100% rename from test/model-caching/python/mpg-estimator/sample.json rename to test/apis/model-caching/python/mpg-estimator/sample.json diff --git a/test/model-caching/tensorflow/multi-model-classifier/README.md b/test/apis/model-caching/tensorflow/multi-model-classifier/README.md similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/README.md rename to test/apis/model-caching/tensorflow/multi-model-classifier/README.md diff --git a/test/model-caching/tensorflow/multi-model-classifier/cortex.yaml b/test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/cortex.yaml rename to test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml diff --git a/test/model-caching/tensorflow/multi-model-classifier/predictor.py b/test/apis/model-caching/tensorflow/multi-model-classifier/predictor.py similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/predictor.py rename to test/apis/model-caching/tensorflow/multi-model-classifier/predictor.py diff --git a/test/model-caching/tensorflow/multi-model-classifier/requirements.txt b/test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/requirements.txt rename to test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt diff --git a/test/model-caching/tensorflow/multi-model-classifier/sample-image.json b/test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/sample-image.json rename to test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json diff --git a/test/model-caching/tensorflow/multi-model-classifier/sample-iris.json b/test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/sample-iris.json rename to test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json diff --git a/test/onnx/iris-classifier/cortex.yaml b/test/apis/onnx/iris-classifier/cortex.yaml similarity index 100% rename from test/onnx/iris-classifier/cortex.yaml rename to test/apis/onnx/iris-classifier/cortex.yaml diff --git a/test/onnx/iris-classifier/predictor.py b/test/apis/onnx/iris-classifier/predictor.py similarity index 100% rename from test/onnx/iris-classifier/predictor.py rename to test/apis/onnx/iris-classifier/predictor.py diff --git a/test/onnx/iris-classifier/sample.json b/test/apis/onnx/iris-classifier/sample.json similarity index 100% rename from test/onnx/iris-classifier/sample.json rename to test/apis/onnx/iris-classifier/sample.json diff --git a/test/onnx/iris-classifier/xgboost.ipynb b/test/apis/onnx/iris-classifier/xgboost.ipynb similarity index 100% rename from test/onnx/iris-classifier/xgboost.ipynb rename to test/apis/onnx/iris-classifier/xgboost.ipynb diff --git a/test/onnx/multi-model-classifier/README.md b/test/apis/onnx/multi-model-classifier/README.md similarity index 100% rename from test/onnx/multi-model-classifier/README.md rename to test/apis/onnx/multi-model-classifier/README.md diff --git a/test/onnx/multi-model-classifier/cortex.yaml b/test/apis/onnx/multi-model-classifier/cortex.yaml similarity index 100% rename from test/onnx/multi-model-classifier/cortex.yaml rename to test/apis/onnx/multi-model-classifier/cortex.yaml diff --git a/test/onnx/multi-model-classifier/predictor.py b/test/apis/onnx/multi-model-classifier/predictor.py similarity index 100% rename from test/onnx/multi-model-classifier/predictor.py rename to test/apis/onnx/multi-model-classifier/predictor.py diff --git a/test/onnx/multi-model-classifier/requirements.txt b/test/apis/onnx/multi-model-classifier/requirements.txt similarity index 100% rename from test/onnx/multi-model-classifier/requirements.txt rename to test/apis/onnx/multi-model-classifier/requirements.txt diff --git a/test/onnx/multi-model-classifier/sample.json b/test/apis/onnx/multi-model-classifier/sample.json similarity index 100% rename from test/onnx/multi-model-classifier/sample.json rename to test/apis/onnx/multi-model-classifier/sample.json diff --git a/test/onnx/yolov5-youtube/README.md b/test/apis/onnx/yolov5-youtube/README.md similarity index 100% rename from test/onnx/yolov5-youtube/README.md rename to test/apis/onnx/yolov5-youtube/README.md diff --git a/test/onnx/yolov5-youtube/conda-packages.txt b/test/apis/onnx/yolov5-youtube/conda-packages.txt similarity index 100% rename from test/onnx/yolov5-youtube/conda-packages.txt rename to test/apis/onnx/yolov5-youtube/conda-packages.txt diff --git a/test/onnx/yolov5-youtube/cortex.yaml b/test/apis/onnx/yolov5-youtube/cortex.yaml similarity index 100% rename from test/onnx/yolov5-youtube/cortex.yaml rename to test/apis/onnx/yolov5-youtube/cortex.yaml diff --git a/test/onnx/yolov5-youtube/labels.json b/test/apis/onnx/yolov5-youtube/labels.json similarity index 100% rename from test/onnx/yolov5-youtube/labels.json rename to test/apis/onnx/yolov5-youtube/labels.json diff --git a/test/onnx/yolov5-youtube/predictor.py b/test/apis/onnx/yolov5-youtube/predictor.py similarity index 100% rename from test/onnx/yolov5-youtube/predictor.py rename to test/apis/onnx/yolov5-youtube/predictor.py diff --git a/test/onnx/yolov5-youtube/requirements.txt b/test/apis/onnx/yolov5-youtube/requirements.txt similarity index 100% rename from test/onnx/yolov5-youtube/requirements.txt rename to test/apis/onnx/yolov5-youtube/requirements.txt diff --git a/test/onnx/yolov5-youtube/sample.json b/test/apis/onnx/yolov5-youtube/sample.json similarity index 100% rename from test/onnx/yolov5-youtube/sample.json rename to test/apis/onnx/yolov5-youtube/sample.json diff --git a/test/onnx/yolov5-youtube/utils.py b/test/apis/onnx/yolov5-youtube/utils.py similarity index 100% rename from test/onnx/yolov5-youtube/utils.py rename to test/apis/onnx/yolov5-youtube/utils.py diff --git a/test/pytorch/answer-generator/cortex.yaml b/test/apis/pytorch/answer-generator/cortex.yaml similarity index 100% rename from test/pytorch/answer-generator/cortex.yaml rename to test/apis/pytorch/answer-generator/cortex.yaml diff --git a/test/pytorch/answer-generator/generator.py b/test/apis/pytorch/answer-generator/generator.py similarity index 100% rename from test/pytorch/answer-generator/generator.py rename to test/apis/pytorch/answer-generator/generator.py diff --git a/test/pytorch/answer-generator/predictor.py b/test/apis/pytorch/answer-generator/predictor.py similarity index 100% rename from test/pytorch/answer-generator/predictor.py rename to test/apis/pytorch/answer-generator/predictor.py diff --git a/test/pytorch/answer-generator/requirements.txt b/test/apis/pytorch/answer-generator/requirements.txt similarity index 100% rename from test/pytorch/answer-generator/requirements.txt rename to test/apis/pytorch/answer-generator/requirements.txt diff --git a/test/pytorch/answer-generator/sample.json b/test/apis/pytorch/answer-generator/sample.json similarity index 100% rename from test/pytorch/answer-generator/sample.json rename to test/apis/pytorch/answer-generator/sample.json diff --git a/test/pytorch/image-classifier-alexnet/cortex.yaml b/test/apis/pytorch/image-classifier-alexnet/cortex.yaml similarity index 100% rename from test/pytorch/image-classifier-alexnet/cortex.yaml rename to test/apis/pytorch/image-classifier-alexnet/cortex.yaml diff --git a/test/pytorch/image-classifier-alexnet/predictor.py b/test/apis/pytorch/image-classifier-alexnet/predictor.py similarity index 100% rename from test/pytorch/image-classifier-alexnet/predictor.py rename to test/apis/pytorch/image-classifier-alexnet/predictor.py diff --git a/test/pytorch/image-classifier-alexnet/requirements.txt b/test/apis/pytorch/image-classifier-alexnet/requirements.txt similarity index 100% rename from test/pytorch/image-classifier-alexnet/requirements.txt rename to test/apis/pytorch/image-classifier-alexnet/requirements.txt diff --git a/test/pytorch/image-classifier-alexnet/sample.json b/test/apis/pytorch/image-classifier-alexnet/sample.json similarity index 100% rename from test/pytorch/image-classifier-alexnet/sample.json rename to test/apis/pytorch/image-classifier-alexnet/sample.json diff --git a/test/pytorch/image-classifier-resnet50/README.md b/test/apis/pytorch/image-classifier-resnet50/README.md similarity index 100% rename from test/pytorch/image-classifier-resnet50/README.md rename to test/apis/pytorch/image-classifier-resnet50/README.md diff --git a/test/pytorch/image-classifier-resnet50/cortex.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex.yaml similarity index 100% rename from test/pytorch/image-classifier-resnet50/cortex.yaml rename to test/apis/pytorch/image-classifier-resnet50/cortex.yaml diff --git a/test/pytorch/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml similarity index 100% rename from test/pytorch/image-classifier-resnet50/cortex_gpu.yaml rename to test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml diff --git a/test/pytorch/image-classifier-resnet50/cortex_inf.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml similarity index 100% rename from test/pytorch/image-classifier-resnet50/cortex_inf.yaml rename to test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml diff --git a/test/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb b/test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb similarity index 100% rename from test/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb rename to test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb diff --git a/test/pytorch/image-classifier-resnet50/predictor.py b/test/apis/pytorch/image-classifier-resnet50/predictor.py similarity index 100% rename from test/pytorch/image-classifier-resnet50/predictor.py rename to test/apis/pytorch/image-classifier-resnet50/predictor.py diff --git a/test/pytorch/image-classifier-resnet50/sample.json b/test/apis/pytorch/image-classifier-resnet50/sample.json similarity index 100% rename from test/pytorch/image-classifier-resnet50/sample.json rename to test/apis/pytorch/image-classifier-resnet50/sample.json diff --git a/test/pytorch/iris-classifier/cortex.yaml b/test/apis/pytorch/iris-classifier/cortex.yaml similarity index 100% rename from test/pytorch/iris-classifier/cortex.yaml rename to test/apis/pytorch/iris-classifier/cortex.yaml diff --git a/test/apis/pytorch/iris-classifier/expectations.yaml b/test/apis/pytorch/iris-classifier/expectations.yaml new file mode 100644 index 0000000000..93b58bac83 --- /dev/null +++ b/test/apis/pytorch/iris-classifier/expectations.yaml @@ -0,0 +1,5 @@ +# this file is used for testing purposes only + +response: + content_type: "text" + expected: "versicolor" diff --git a/test/pytorch/iris-classifier/model.py b/test/apis/pytorch/iris-classifier/model.py similarity index 100% rename from test/pytorch/iris-classifier/model.py rename to test/apis/pytorch/iris-classifier/model.py diff --git a/test/pytorch/iris-classifier/predictor.py b/test/apis/pytorch/iris-classifier/predictor.py similarity index 100% rename from test/pytorch/iris-classifier/predictor.py rename to test/apis/pytorch/iris-classifier/predictor.py diff --git a/test/pytorch/iris-classifier/requirements.txt b/test/apis/pytorch/iris-classifier/requirements.txt similarity index 100% rename from test/pytorch/iris-classifier/requirements.txt rename to test/apis/pytorch/iris-classifier/requirements.txt diff --git a/test/pytorch/iris-classifier/sample.json b/test/apis/pytorch/iris-classifier/sample.json similarity index 100% rename from test/pytorch/iris-classifier/sample.json rename to test/apis/pytorch/iris-classifier/sample.json diff --git a/test/pytorch/language-identifier/cortex.yaml b/test/apis/pytorch/language-identifier/cortex.yaml similarity index 100% rename from test/pytorch/language-identifier/cortex.yaml rename to test/apis/pytorch/language-identifier/cortex.yaml diff --git a/test/pytorch/language-identifier/predictor.py b/test/apis/pytorch/language-identifier/predictor.py similarity index 100% rename from test/pytorch/language-identifier/predictor.py rename to test/apis/pytorch/language-identifier/predictor.py diff --git a/test/pytorch/language-identifier/requirements.txt b/test/apis/pytorch/language-identifier/requirements.txt similarity index 100% rename from test/pytorch/language-identifier/requirements.txt rename to test/apis/pytorch/language-identifier/requirements.txt diff --git a/test/pytorch/language-identifier/sample.json b/test/apis/pytorch/language-identifier/sample.json similarity index 100% rename from test/pytorch/language-identifier/sample.json rename to test/apis/pytorch/language-identifier/sample.json diff --git a/test/pytorch/multi-model-text-analyzer/README.md b/test/apis/pytorch/multi-model-text-analyzer/README.md similarity index 100% rename from test/pytorch/multi-model-text-analyzer/README.md rename to test/apis/pytorch/multi-model-text-analyzer/README.md diff --git a/test/pytorch/multi-model-text-analyzer/cortex.yaml b/test/apis/pytorch/multi-model-text-analyzer/cortex.yaml similarity index 100% rename from test/pytorch/multi-model-text-analyzer/cortex.yaml rename to test/apis/pytorch/multi-model-text-analyzer/cortex.yaml diff --git a/test/pytorch/multi-model-text-analyzer/predictor.py b/test/apis/pytorch/multi-model-text-analyzer/predictor.py similarity index 100% rename from test/pytorch/multi-model-text-analyzer/predictor.py rename to test/apis/pytorch/multi-model-text-analyzer/predictor.py diff --git a/test/pytorch/multi-model-text-analyzer/requirements.txt b/test/apis/pytorch/multi-model-text-analyzer/requirements.txt similarity index 100% rename from test/pytorch/multi-model-text-analyzer/requirements.txt rename to test/apis/pytorch/multi-model-text-analyzer/requirements.txt diff --git a/test/pytorch/multi-model-text-analyzer/sample-sentiment.json b/test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json similarity index 100% rename from test/pytorch/multi-model-text-analyzer/sample-sentiment.json rename to test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json diff --git a/test/pytorch/multi-model-text-analyzer/sample-summarizer.json b/test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json similarity index 100% rename from test/pytorch/multi-model-text-analyzer/sample-summarizer.json rename to test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json diff --git a/test/pytorch/object-detector/coco_labels.txt b/test/apis/pytorch/object-detector/coco_labels.txt similarity index 100% rename from test/pytorch/object-detector/coco_labels.txt rename to test/apis/pytorch/object-detector/coco_labels.txt diff --git a/test/pytorch/object-detector/cortex.yaml b/test/apis/pytorch/object-detector/cortex.yaml similarity index 100% rename from test/pytorch/object-detector/cortex.yaml rename to test/apis/pytorch/object-detector/cortex.yaml diff --git a/test/pytorch/object-detector/predictor.py b/test/apis/pytorch/object-detector/predictor.py similarity index 100% rename from test/pytorch/object-detector/predictor.py rename to test/apis/pytorch/object-detector/predictor.py diff --git a/test/pytorch/object-detector/requirements.txt b/test/apis/pytorch/object-detector/requirements.txt similarity index 100% rename from test/pytorch/object-detector/requirements.txt rename to test/apis/pytorch/object-detector/requirements.txt diff --git a/test/pytorch/object-detector/sample.json b/test/apis/pytorch/object-detector/sample.json similarity index 100% rename from test/pytorch/object-detector/sample.json rename to test/apis/pytorch/object-detector/sample.json diff --git a/test/pytorch/question-generator/cortex.yaml b/test/apis/pytorch/question-generator/cortex.yaml similarity index 100% rename from test/pytorch/question-generator/cortex.yaml rename to test/apis/pytorch/question-generator/cortex.yaml diff --git a/test/pytorch/question-generator/dependencies.sh b/test/apis/pytorch/question-generator/dependencies.sh similarity index 100% rename from test/pytorch/question-generator/dependencies.sh rename to test/apis/pytorch/question-generator/dependencies.sh diff --git a/test/pytorch/question-generator/predictor.py b/test/apis/pytorch/question-generator/predictor.py similarity index 100% rename from test/pytorch/question-generator/predictor.py rename to test/apis/pytorch/question-generator/predictor.py diff --git a/test/pytorch/question-generator/requirements.txt b/test/apis/pytorch/question-generator/requirements.txt similarity index 100% rename from test/pytorch/question-generator/requirements.txt rename to test/apis/pytorch/question-generator/requirements.txt diff --git a/test/pytorch/question-generator/sample.json b/test/apis/pytorch/question-generator/sample.json similarity index 100% rename from test/pytorch/question-generator/sample.json rename to test/apis/pytorch/question-generator/sample.json diff --git a/test/pytorch/reading-comprehender/cortex.yaml b/test/apis/pytorch/reading-comprehender/cortex.yaml similarity index 100% rename from test/pytorch/reading-comprehender/cortex.yaml rename to test/apis/pytorch/reading-comprehender/cortex.yaml diff --git a/test/pytorch/reading-comprehender/predictor.py b/test/apis/pytorch/reading-comprehender/predictor.py similarity index 100% rename from test/pytorch/reading-comprehender/predictor.py rename to test/apis/pytorch/reading-comprehender/predictor.py diff --git a/test/pytorch/reading-comprehender/requirements.txt b/test/apis/pytorch/reading-comprehender/requirements.txt similarity index 100% rename from test/pytorch/reading-comprehender/requirements.txt rename to test/apis/pytorch/reading-comprehender/requirements.txt diff --git a/test/pytorch/reading-comprehender/sample.json b/test/apis/pytorch/reading-comprehender/sample.json similarity index 100% rename from test/pytorch/reading-comprehender/sample.json rename to test/apis/pytorch/reading-comprehender/sample.json diff --git a/test/pytorch/search-completer/cortex.yaml b/test/apis/pytorch/search-completer/cortex.yaml similarity index 100% rename from test/pytorch/search-completer/cortex.yaml rename to test/apis/pytorch/search-completer/cortex.yaml diff --git a/test/pytorch/search-completer/predictor.py b/test/apis/pytorch/search-completer/predictor.py similarity index 100% rename from test/pytorch/search-completer/predictor.py rename to test/apis/pytorch/search-completer/predictor.py diff --git a/test/pytorch/search-completer/requirements.txt b/test/apis/pytorch/search-completer/requirements.txt similarity index 100% rename from test/pytorch/search-completer/requirements.txt rename to test/apis/pytorch/search-completer/requirements.txt diff --git a/test/pytorch/search-completer/sample.json b/test/apis/pytorch/search-completer/sample.json similarity index 100% rename from test/pytorch/search-completer/sample.json rename to test/apis/pytorch/search-completer/sample.json diff --git a/test/pytorch/sentiment-analyzer/cortex.yaml b/test/apis/pytorch/sentiment-analyzer/cortex.yaml similarity index 100% rename from test/pytorch/sentiment-analyzer/cortex.yaml rename to test/apis/pytorch/sentiment-analyzer/cortex.yaml diff --git a/test/pytorch/sentiment-analyzer/predictor.py b/test/apis/pytorch/sentiment-analyzer/predictor.py similarity index 100% rename from test/pytorch/sentiment-analyzer/predictor.py rename to test/apis/pytorch/sentiment-analyzer/predictor.py diff --git a/test/pytorch/sentiment-analyzer/requirements.txt b/test/apis/pytorch/sentiment-analyzer/requirements.txt similarity index 100% rename from test/pytorch/sentiment-analyzer/requirements.txt rename to test/apis/pytorch/sentiment-analyzer/requirements.txt diff --git a/test/pytorch/sentiment-analyzer/sample.json b/test/apis/pytorch/sentiment-analyzer/sample.json similarity index 100% rename from test/pytorch/sentiment-analyzer/sample.json rename to test/apis/pytorch/sentiment-analyzer/sample.json diff --git a/test/pytorch/text-generator/README.md b/test/apis/pytorch/text-generator/README.md similarity index 100% rename from test/pytorch/text-generator/README.md rename to test/apis/pytorch/text-generator/README.md diff --git a/test/pytorch/text-generator/cortex.yaml b/test/apis/pytorch/text-generator/cortex.yaml similarity index 100% rename from test/pytorch/text-generator/cortex.yaml rename to test/apis/pytorch/text-generator/cortex.yaml diff --git a/test/pytorch/text-generator/deploy.ipynb b/test/apis/pytorch/text-generator/deploy.ipynb similarity index 100% rename from test/pytorch/text-generator/deploy.ipynb rename to test/apis/pytorch/text-generator/deploy.ipynb diff --git a/test/pytorch/text-generator/predictor.py b/test/apis/pytorch/text-generator/predictor.py similarity index 100% rename from test/pytorch/text-generator/predictor.py rename to test/apis/pytorch/text-generator/predictor.py diff --git a/test/pytorch/text-generator/requirements.txt b/test/apis/pytorch/text-generator/requirements.txt similarity index 100% rename from test/pytorch/text-generator/requirements.txt rename to test/apis/pytorch/text-generator/requirements.txt diff --git a/test/pytorch/text-generator/sample.json b/test/apis/pytorch/text-generator/sample.json similarity index 100% rename from test/pytorch/text-generator/sample.json rename to test/apis/pytorch/text-generator/sample.json diff --git a/test/pytorch/text-summarizer/README.md b/test/apis/pytorch/text-summarizer/README.md similarity index 100% rename from test/pytorch/text-summarizer/README.md rename to test/apis/pytorch/text-summarizer/README.md diff --git a/test/pytorch/text-summarizer/cortex.yaml b/test/apis/pytorch/text-summarizer/cortex.yaml similarity index 100% rename from test/pytorch/text-summarizer/cortex.yaml rename to test/apis/pytorch/text-summarizer/cortex.yaml diff --git a/test/pytorch/text-summarizer/predictor.py b/test/apis/pytorch/text-summarizer/predictor.py similarity index 100% rename from test/pytorch/text-summarizer/predictor.py rename to test/apis/pytorch/text-summarizer/predictor.py diff --git a/test/pytorch/text-summarizer/requirements.txt b/test/apis/pytorch/text-summarizer/requirements.txt similarity index 100% rename from test/pytorch/text-summarizer/requirements.txt rename to test/apis/pytorch/text-summarizer/requirements.txt diff --git a/test/pytorch/text-summarizer/sample.json b/test/apis/pytorch/text-summarizer/sample.json similarity index 100% rename from test/pytorch/text-summarizer/sample.json rename to test/apis/pytorch/text-summarizer/sample.json diff --git a/test/sklearn/iris-classifier/cortex.yaml b/test/apis/sklearn/iris-classifier/cortex.yaml similarity index 100% rename from test/sklearn/iris-classifier/cortex.yaml rename to test/apis/sklearn/iris-classifier/cortex.yaml diff --git a/test/sklearn/iris-classifier/predictor.py b/test/apis/sklearn/iris-classifier/predictor.py similarity index 100% rename from test/sklearn/iris-classifier/predictor.py rename to test/apis/sklearn/iris-classifier/predictor.py diff --git a/test/sklearn/iris-classifier/requirements.txt b/test/apis/sklearn/iris-classifier/requirements.txt similarity index 100% rename from test/sklearn/iris-classifier/requirements.txt rename to test/apis/sklearn/iris-classifier/requirements.txt diff --git a/test/sklearn/iris-classifier/sample.json b/test/apis/sklearn/iris-classifier/sample.json similarity index 100% rename from test/sklearn/iris-classifier/sample.json rename to test/apis/sklearn/iris-classifier/sample.json diff --git a/test/sklearn/iris-classifier/trainer.py b/test/apis/sklearn/iris-classifier/trainer.py similarity index 100% rename from test/sklearn/iris-classifier/trainer.py rename to test/apis/sklearn/iris-classifier/trainer.py diff --git a/test/sklearn/mpg-estimator/cortex.yaml b/test/apis/sklearn/mpg-estimator/cortex.yaml similarity index 100% rename from test/sklearn/mpg-estimator/cortex.yaml rename to test/apis/sklearn/mpg-estimator/cortex.yaml diff --git a/test/sklearn/mpg-estimator/predictor.py b/test/apis/sklearn/mpg-estimator/predictor.py similarity index 100% rename from test/sklearn/mpg-estimator/predictor.py rename to test/apis/sklearn/mpg-estimator/predictor.py diff --git a/test/sklearn/mpg-estimator/requirements.txt b/test/apis/sklearn/mpg-estimator/requirements.txt similarity index 100% rename from test/sklearn/mpg-estimator/requirements.txt rename to test/apis/sklearn/mpg-estimator/requirements.txt diff --git a/test/sklearn/mpg-estimator/sample.json b/test/apis/sklearn/mpg-estimator/sample.json similarity index 100% rename from test/sklearn/mpg-estimator/sample.json rename to test/apis/sklearn/mpg-estimator/sample.json diff --git a/test/sklearn/mpg-estimator/trainer.py b/test/apis/sklearn/mpg-estimator/trainer.py similarity index 100% rename from test/sklearn/mpg-estimator/trainer.py rename to test/apis/sklearn/mpg-estimator/trainer.py diff --git a/test/spacy/entity-recognizer/cortex.yaml b/test/apis/spacy/entity-recognizer/cortex.yaml similarity index 100% rename from test/spacy/entity-recognizer/cortex.yaml rename to test/apis/spacy/entity-recognizer/cortex.yaml diff --git a/test/spacy/entity-recognizer/predictor.py b/test/apis/spacy/entity-recognizer/predictor.py similarity index 100% rename from test/spacy/entity-recognizer/predictor.py rename to test/apis/spacy/entity-recognizer/predictor.py diff --git a/test/spacy/entity-recognizer/requirements.txt b/test/apis/spacy/entity-recognizer/requirements.txt similarity index 100% rename from test/spacy/entity-recognizer/requirements.txt rename to test/apis/spacy/entity-recognizer/requirements.txt diff --git a/test/spacy/entity-recognizer/sample.json b/test/apis/spacy/entity-recognizer/sample.json similarity index 100% rename from test/spacy/entity-recognizer/sample.json rename to test/apis/spacy/entity-recognizer/sample.json diff --git a/test/tensorflow/image-classifier-inception/cortex.yaml b/test/apis/tensorflow/image-classifier-inception/cortex.yaml similarity index 100% rename from test/tensorflow/image-classifier-inception/cortex.yaml rename to test/apis/tensorflow/image-classifier-inception/cortex.yaml diff --git a/test/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml similarity index 100% rename from test/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml rename to test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml diff --git a/test/tensorflow/image-classifier-inception/inception.ipynb b/test/apis/tensorflow/image-classifier-inception/inception.ipynb similarity index 100% rename from test/tensorflow/image-classifier-inception/inception.ipynb rename to test/apis/tensorflow/image-classifier-inception/inception.ipynb diff --git a/test/tensorflow/image-classifier-inception/predictor.py b/test/apis/tensorflow/image-classifier-inception/predictor.py similarity index 100% rename from test/tensorflow/image-classifier-inception/predictor.py rename to test/apis/tensorflow/image-classifier-inception/predictor.py diff --git a/test/tensorflow/image-classifier-inception/requirements.txt b/test/apis/tensorflow/image-classifier-inception/requirements.txt similarity index 100% rename from test/tensorflow/image-classifier-inception/requirements.txt rename to test/apis/tensorflow/image-classifier-inception/requirements.txt diff --git a/test/tensorflow/image-classifier-inception/sample.json b/test/apis/tensorflow/image-classifier-inception/sample.json similarity index 100% rename from test/tensorflow/image-classifier-inception/sample.json rename to test/apis/tensorflow/image-classifier-inception/sample.json diff --git a/test/tensorflow/image-classifier-resnet50/README.md b/test/apis/tensorflow/image-classifier-resnet50/README.md similarity index 100% rename from test/tensorflow/image-classifier-resnet50/README.md rename to test/apis/tensorflow/image-classifier-resnet50/README.md diff --git a/test/tensorflow/image-classifier-resnet50/cortex.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_gpu.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_inf.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_inf.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml diff --git a/test/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb b/test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb similarity index 100% rename from test/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb rename to test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb diff --git a/test/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb b/test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb similarity index 100% rename from test/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb rename to test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb diff --git a/test/tensorflow/image-classifier-resnet50/predictor.py b/test/apis/tensorflow/image-classifier-resnet50/predictor.py similarity index 100% rename from test/tensorflow/image-classifier-resnet50/predictor.py rename to test/apis/tensorflow/image-classifier-resnet50/predictor.py diff --git a/test/tensorflow/image-classifier-resnet50/requirements.txt b/test/apis/tensorflow/image-classifier-resnet50/requirements.txt similarity index 100% rename from test/tensorflow/image-classifier-resnet50/requirements.txt rename to test/apis/tensorflow/image-classifier-resnet50/requirements.txt diff --git a/test/tensorflow/image-classifier-resnet50/sample.bin b/test/apis/tensorflow/image-classifier-resnet50/sample.bin similarity index 100% rename from test/tensorflow/image-classifier-resnet50/sample.bin rename to test/apis/tensorflow/image-classifier-resnet50/sample.bin diff --git a/test/tensorflow/image-classifier-resnet50/sample.json b/test/apis/tensorflow/image-classifier-resnet50/sample.json similarity index 100% rename from test/tensorflow/image-classifier-resnet50/sample.json rename to test/apis/tensorflow/image-classifier-resnet50/sample.json diff --git a/test/tensorflow/iris-classifier/cortex.yaml b/test/apis/tensorflow/iris-classifier/cortex.yaml similarity index 100% rename from test/tensorflow/iris-classifier/cortex.yaml rename to test/apis/tensorflow/iris-classifier/cortex.yaml diff --git a/test/tensorflow/iris-classifier/predictor.py b/test/apis/tensorflow/iris-classifier/predictor.py similarity index 100% rename from test/tensorflow/iris-classifier/predictor.py rename to test/apis/tensorflow/iris-classifier/predictor.py diff --git a/test/tensorflow/iris-classifier/sample.json b/test/apis/tensorflow/iris-classifier/sample.json similarity index 100% rename from test/tensorflow/iris-classifier/sample.json rename to test/apis/tensorflow/iris-classifier/sample.json diff --git a/test/tensorflow/iris-classifier/tensorflow.ipynb b/test/apis/tensorflow/iris-classifier/tensorflow.ipynb similarity index 100% rename from test/tensorflow/iris-classifier/tensorflow.ipynb rename to test/apis/tensorflow/iris-classifier/tensorflow.ipynb diff --git a/test/tensorflow/license-plate-reader/README.md b/test/apis/tensorflow/license-plate-reader/README.md similarity index 100% rename from test/tensorflow/license-plate-reader/README.md rename to test/apis/tensorflow/license-plate-reader/README.md diff --git a/test/tensorflow/license-plate-reader/config.json b/test/apis/tensorflow/license-plate-reader/config.json similarity index 100% rename from test/tensorflow/license-plate-reader/config.json rename to test/apis/tensorflow/license-plate-reader/config.json diff --git a/test/tensorflow/license-plate-reader/cortex_full.yaml b/test/apis/tensorflow/license-plate-reader/cortex_full.yaml similarity index 100% rename from test/tensorflow/license-plate-reader/cortex_full.yaml rename to test/apis/tensorflow/license-plate-reader/cortex_full.yaml diff --git a/test/tensorflow/license-plate-reader/cortex_lite.yaml b/test/apis/tensorflow/license-plate-reader/cortex_lite.yaml similarity index 100% rename from test/tensorflow/license-plate-reader/cortex_lite.yaml rename to test/apis/tensorflow/license-plate-reader/cortex_lite.yaml diff --git a/test/tensorflow/license-plate-reader/predictor_crnn.py b/test/apis/tensorflow/license-plate-reader/predictor_crnn.py similarity index 100% rename from test/tensorflow/license-plate-reader/predictor_crnn.py rename to test/apis/tensorflow/license-plate-reader/predictor_crnn.py diff --git a/test/tensorflow/license-plate-reader/predictor_lite.py b/test/apis/tensorflow/license-plate-reader/predictor_lite.py similarity index 100% rename from test/tensorflow/license-plate-reader/predictor_lite.py rename to test/apis/tensorflow/license-plate-reader/predictor_lite.py diff --git a/test/tensorflow/license-plate-reader/predictor_yolo.py b/test/apis/tensorflow/license-plate-reader/predictor_yolo.py similarity index 100% rename from test/tensorflow/license-plate-reader/predictor_yolo.py rename to test/apis/tensorflow/license-plate-reader/predictor_yolo.py diff --git a/test/tensorflow/license-plate-reader/requirements.txt b/test/apis/tensorflow/license-plate-reader/requirements.txt similarity index 100% rename from test/tensorflow/license-plate-reader/requirements.txt rename to test/apis/tensorflow/license-plate-reader/requirements.txt diff --git a/test/tensorflow/license-plate-reader/sample_inference.py b/test/apis/tensorflow/license-plate-reader/sample_inference.py similarity index 100% rename from test/tensorflow/license-plate-reader/sample_inference.py rename to test/apis/tensorflow/license-plate-reader/sample_inference.py diff --git a/test/tensorflow/license-plate-reader/utils/__init__.py b/test/apis/tensorflow/license-plate-reader/utils/__init__.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/__init__.py rename to test/apis/tensorflow/license-plate-reader/utils/__init__.py diff --git a/test/tensorflow/license-plate-reader/utils/bbox.py b/test/apis/tensorflow/license-plate-reader/utils/bbox.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/bbox.py rename to test/apis/tensorflow/license-plate-reader/utils/bbox.py diff --git a/test/tensorflow/license-plate-reader/utils/colors.py b/test/apis/tensorflow/license-plate-reader/utils/colors.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/colors.py rename to test/apis/tensorflow/license-plate-reader/utils/colors.py diff --git a/test/tensorflow/license-plate-reader/utils/preprocess.py b/test/apis/tensorflow/license-plate-reader/utils/preprocess.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/preprocess.py rename to test/apis/tensorflow/license-plate-reader/utils/preprocess.py diff --git a/test/tensorflow/license-plate-reader/utils/utils.py b/test/apis/tensorflow/license-plate-reader/utils/utils.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/utils.py rename to test/apis/tensorflow/license-plate-reader/utils/utils.py diff --git a/test/tensorflow/multi-model-classifier/README.md b/test/apis/tensorflow/multi-model-classifier/README.md similarity index 100% rename from test/tensorflow/multi-model-classifier/README.md rename to test/apis/tensorflow/multi-model-classifier/README.md diff --git a/test/tensorflow/multi-model-classifier/cortex.yaml b/test/apis/tensorflow/multi-model-classifier/cortex.yaml similarity index 100% rename from test/tensorflow/multi-model-classifier/cortex.yaml rename to test/apis/tensorflow/multi-model-classifier/cortex.yaml diff --git a/test/tensorflow/multi-model-classifier/predictor.py b/test/apis/tensorflow/multi-model-classifier/predictor.py similarity index 100% rename from test/tensorflow/multi-model-classifier/predictor.py rename to test/apis/tensorflow/multi-model-classifier/predictor.py diff --git a/test/tensorflow/multi-model-classifier/requirements.txt b/test/apis/tensorflow/multi-model-classifier/requirements.txt similarity index 100% rename from test/tensorflow/multi-model-classifier/requirements.txt rename to test/apis/tensorflow/multi-model-classifier/requirements.txt diff --git a/test/tensorflow/multi-model-classifier/sample-image.json b/test/apis/tensorflow/multi-model-classifier/sample-image.json similarity index 100% rename from test/tensorflow/multi-model-classifier/sample-image.json rename to test/apis/tensorflow/multi-model-classifier/sample-image.json diff --git a/test/tensorflow/multi-model-classifier/sample-iris.json b/test/apis/tensorflow/multi-model-classifier/sample-iris.json similarity index 100% rename from test/tensorflow/multi-model-classifier/sample-iris.json rename to test/apis/tensorflow/multi-model-classifier/sample-iris.json diff --git a/test/tensorflow/sentiment-analyzer/bert.ipynb b/test/apis/tensorflow/sentiment-analyzer/bert.ipynb similarity index 100% rename from test/tensorflow/sentiment-analyzer/bert.ipynb rename to test/apis/tensorflow/sentiment-analyzer/bert.ipynb diff --git a/test/tensorflow/sentiment-analyzer/cortex.yaml b/test/apis/tensorflow/sentiment-analyzer/cortex.yaml similarity index 100% rename from test/tensorflow/sentiment-analyzer/cortex.yaml rename to test/apis/tensorflow/sentiment-analyzer/cortex.yaml diff --git a/test/tensorflow/sentiment-analyzer/predictor.py b/test/apis/tensorflow/sentiment-analyzer/predictor.py similarity index 100% rename from test/tensorflow/sentiment-analyzer/predictor.py rename to test/apis/tensorflow/sentiment-analyzer/predictor.py diff --git a/test/tensorflow/sentiment-analyzer/requirements.txt b/test/apis/tensorflow/sentiment-analyzer/requirements.txt similarity index 100% rename from test/tensorflow/sentiment-analyzer/requirements.txt rename to test/apis/tensorflow/sentiment-analyzer/requirements.txt diff --git a/test/tensorflow/sentiment-analyzer/sample.json b/test/apis/tensorflow/sentiment-analyzer/sample.json similarity index 100% rename from test/tensorflow/sentiment-analyzer/sample.json rename to test/apis/tensorflow/sentiment-analyzer/sample.json diff --git a/test/tensorflow/text-generator/cortex.yaml b/test/apis/tensorflow/text-generator/cortex.yaml similarity index 100% rename from test/tensorflow/text-generator/cortex.yaml rename to test/apis/tensorflow/text-generator/cortex.yaml diff --git a/test/tensorflow/text-generator/encoder.py b/test/apis/tensorflow/text-generator/encoder.py similarity index 100% rename from test/tensorflow/text-generator/encoder.py rename to test/apis/tensorflow/text-generator/encoder.py diff --git a/test/tensorflow/text-generator/gpt-2.ipynb b/test/apis/tensorflow/text-generator/gpt-2.ipynb similarity index 100% rename from test/tensorflow/text-generator/gpt-2.ipynb rename to test/apis/tensorflow/text-generator/gpt-2.ipynb diff --git a/test/tensorflow/text-generator/predictor.py b/test/apis/tensorflow/text-generator/predictor.py similarity index 100% rename from test/tensorflow/text-generator/predictor.py rename to test/apis/tensorflow/text-generator/predictor.py diff --git a/test/tensorflow/text-generator/requirements.txt b/test/apis/tensorflow/text-generator/requirements.txt similarity index 100% rename from test/tensorflow/text-generator/requirements.txt rename to test/apis/tensorflow/text-generator/requirements.txt diff --git a/test/tensorflow/text-generator/sample.json b/test/apis/tensorflow/text-generator/sample.json similarity index 100% rename from test/tensorflow/text-generator/sample.json rename to test/apis/tensorflow/text-generator/sample.json diff --git a/test/traffic-splitter/README.md b/test/apis/traffic-splitter/README.md similarity index 100% rename from test/traffic-splitter/README.md rename to test/apis/traffic-splitter/README.md diff --git a/test/traffic-splitter/cortex.yaml b/test/apis/traffic-splitter/cortex.yaml similarity index 100% rename from test/traffic-splitter/cortex.yaml rename to test/apis/traffic-splitter/cortex.yaml diff --git a/test/traffic-splitter/model.py b/test/apis/traffic-splitter/model.py similarity index 100% rename from test/traffic-splitter/model.py rename to test/apis/traffic-splitter/model.py diff --git a/test/traffic-splitter/onnx_predictor.py b/test/apis/traffic-splitter/onnx_predictor.py similarity index 100% rename from test/traffic-splitter/onnx_predictor.py rename to test/apis/traffic-splitter/onnx_predictor.py diff --git a/test/traffic-splitter/pytorch_predictor.py b/test/apis/traffic-splitter/pytorch_predictor.py similarity index 100% rename from test/traffic-splitter/pytorch_predictor.py rename to test/apis/traffic-splitter/pytorch_predictor.py diff --git a/test/traffic-splitter/sample.json b/test/apis/traffic-splitter/sample.json similarity index 100% rename from test/traffic-splitter/sample.json rename to test/apis/traffic-splitter/sample.json diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000000..3edf57d2d1 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,75 @@ +# End-to-end Tests + +## Dependencies + +Install the `e2e` package, from the project directory: + +```shell +pip install -e test/e2e +``` + +This only needs to be installed once (not on every code change). + +## Running the tests + +Before running tests, instruct the Python client to use your development CLI binary: + +```shell +export CORTEX_CLI_PATH=/bin/cortex +``` + +### AWS + +From an existing cluster: + +```shell +pytest test/e2e/tests -k aws --aws-env +``` + +Using a new cluster, created for testing only and deleted afterwards: + +```shell +pytest test/e2e/tests -k aws --aws-config +``` + +**Note:** For the BatchAPI tests, the `--s3-bucket` option should be provided with an +AWS S3 bucket for testing purposes. It is more convinient however to define +this bucket through an environment variable, see [configuration](#configuration). + +### GCP + +From an existing cluster: + +```shell +pytest test/e2e/tests -k gcp --gcp-env +``` + +Using a new cluster, created for testing only and deleted afterwards: + +```shell +pytest test/e2e/tests -k gcp --gcp-config +``` + +### All Tests + +You can run all tests at once, however the provider specific options should be passed +accordingly, or the test cases will be skipped. + +e.g. + +```shell +pytest test/e2e/tests --aws-env --gcp-env +``` + +## Configuration + +It is possible to configure the behaviour of the tests by defining +environment variables or a `.env` file at the project directory. + +```dotenv +# .env file +CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT=60 +CORTEX_TEST_BATCH_DEPLOY_TIMEOUT=30 +CORTEX_TEST_BATCH_JOB_TIMEOUT=120 +CORTEX_TEST_BATCH_S3_BUCKET_DIR=s3:///test/jobs +``` diff --git a/test/e2e/e2e/__init__.py b/test/e2e/e2e/__init__.py new file mode 100644 index 0000000000..181f818dd1 --- /dev/null +++ b/test/e2e/e2e/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cluster import create_cluster, delete_cluster + +__all__ = ["create_cluster", "delete_cluster"] diff --git a/test/e2e/e2e/cluster.py b/test/e2e/e2e/cluster.py new file mode 100644 index 0000000000..f85217b7d0 --- /dev/null +++ b/test/e2e/e2e/cluster.py @@ -0,0 +1,59 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys + +import yaml + +from e2e.exceptions import ClusterCreationException, ClusterDeletionException + + +def create_cluster(cluster_config: str): + """Create a cortex cluster from a cluster config""" + with open(cluster_config) as f: + config = yaml.safe_load(f) + + cluster_name = config["cluster_name"] + provider = config["provider"] + + p = subprocess.run( + [ + "cortex", + "cluster", + "up", + "-y", + "--config", + cluster_config, + "--configure-env", + f"{cluster_name}-{provider}", + ], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + if p.returncode != 0: + raise ClusterCreationException(f"failed to create cluster with config: {cluster_config}") + + +def delete_cluster(cluster_config: str): + """Delete a cortex cluster from a cluster config""" + p = subprocess.run( + ["cortex", "cluster", "down", "-y", "--config", cluster_config], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + if p.returncode != 0: + raise ClusterDeletionException(f"failed to delete cluster with config: {cluster_config}") diff --git a/test/e2e/e2e/exceptions.py b/test/e2e/e2e/exceptions.py new file mode 100644 index 0000000000..cc5710290b --- /dev/null +++ b/test/e2e/e2e/exceptions.py @@ -0,0 +1,10 @@ +class ClusterCreationException(Exception): + pass + + +class ClusterDeletionException(Exception): + pass + + +class ExpectationsValidationException(Exception): + pass diff --git a/test/e2e/e2e/expectations.py b/test/e2e/e2e/expectations.py new file mode 100644 index 0000000000..ca88d061bf --- /dev/null +++ b/test/e2e/e2e/expectations.py @@ -0,0 +1,83 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import types +from typing import Dict, Any + +import jsonschema +import requests +import yaml +from jsonschema import Draft7Validator + +from e2e.exceptions import ExpectationsValidationException + +CONTENT_TO_ATTR = {"text": "text", "json": "json", "binary": "content"} + + +def assert_response_expectations(response: requests.Response, expectations: Dict[str, Any]): + content_type = expectations["content_type"] + + expected = expectations.get("expected") + if expected: + output = _get_response_content(response, content_type) + assert output == expected, f"unexpected response: got {output}, expected {expected}" + + expected_json_schema = expectations.get("json_schema") + if expected_json_schema: + output = _get_response_content(response, content_type) + jsonschema.validate(output, schema=expected_json_schema) + + +def parse_expectations(expectations_file: str) -> Dict[str, Any]: + with open(expectations_file) as f: + expectations = yaml.safe_load(f) + + validate_expectations(expectations) + + return expectations + + +def validate_expectations(expectations): + if "response" in expectations: + validate_response_expectations(expectations["response"]) + + +def validate_response_expectations(expectations: Dict[str, Any]): + if not expectations["content_type"] in CONTENT_TO_ATTR.keys(): + raise ExpectationsValidationException( + f"response.content_type should be one of {CONTENT_TO_ATTR.keys()}" + ) + + if "expected" in expectations and "json_schema" in expectations: + raise ExpectationsValidationException("expected and json_schema are mutually exclusive") + + if "json_schema" in expectations: + if expectations["content_type"] != "json": + raise ExpectationsValidationException( + "json_schema is only valid when content_type is set to json" + ) + + try: + Draft7Validator.check_schema(schema=expectations["json_schema"]) + except Exception as e: + raise ExpectationsValidationException("json_schema is invalid") from e + + +def _get_response_content(response: requests.Response, content_type: str) -> str: + attr = CONTENT_TO_ATTR.get(content_type, "content") + content = getattr(response, attr) + if isinstance(content, types.MethodType): + return content() + + return content diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py new file mode 100644 index 0000000000..cac1c44bce --- /dev/null +++ b/test/e2e/e2e/tests.py @@ -0,0 +1,128 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import time +from http import HTTPStatus +from pathlib import Path +from typing import List + +import cortex as cx +import yaml + +from e2e.expectations import parse_expectations, assert_response_expectations +from e2e.utils import ( + apis_ready, + endpoint_ready, + request_prediction, + job_done, + request_batch_prediction, +) + +TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" + + +def delete_apis(client: cx.Client, api_names: List[str]): + for name in api_names: + client.delete_api(name) + + +def test_realtime_api(client: cx.Client, api: str, timeout: int = None): + api_dir = TEST_APIS_DIR / api + with open(str(api_dir / "cortex.yaml")) as f: + api_specs = yaml.safe_load(f) + + expectations = None + expectations_file = api_dir / "expectations.yaml" + if expectations_file.exists(): + expectations = parse_expectations(str(expectations_file)) + + api_name = api_specs[0]["name"] + for api_spec in api_specs: + client.create_api(api_spec=api_spec, project_dir=api_dir) + + try: + assert apis_ready( + client=client, api_names=[api_name], timeout=timeout + ), f"apis {api_name} not ready" + + with open(str(api_dir / "sample.json")) as f: + payload = json.load(f) + + response = request_prediction(client, api_name, payload) + + assert ( + response.status_code == HTTPStatus.OK + ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" + + if expectations and "response" in expectations: + assert_response_expectations(response, expectations["response"]) + finally: + delete_apis(client, [api_name]) + + +def test_batch_api( + client: cx.Client, + api: str, + test_bucket: str, + deploy_timeout: int = None, + job_timeout: int = None, + retry_attempts: int = 0, +): + api_dir = TEST_APIS_DIR / api + with open(str(api_dir / "cortex.yaml")) as f: + api_specs = yaml.safe_load(f) + + assert len(api_specs) == 1 + + api_name = api_specs[0]["name"] + client.create_api(api_spec=api_specs[0], project_dir=api_dir) + + try: + assert endpoint_ready( + client=client, api_name=api_name, timeout=deploy_timeout + ), f"api {api_name} not ready" + + with open(str(api_dir / "sample.json")) as f: + payload = json.load(f) + + for i in range(retry_attempts + 1): + response = request_batch_prediction( + client, + api_name, + item_list=payload, + batch_size=2, + config={"dest_s3_dir": test_bucket}, + ) + if response.status_code == HTTPStatus.OK: + break + + time.sleep(1) + + assert ( + response.status_code == HTTPStatus.OK + ), f"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})" + + job_spec = response.json() + + # monitor job progress + assert job_done( + client=client, + api_name=job_spec["api_name"], + job_id=job_spec["job_id"], + timeout=job_timeout, + ), f"job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})" + + finally: + delete_apis(client, [api_name]) diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py new file mode 100644 index 0000000000..a21ea8d7f7 --- /dev/null +++ b/test/e2e/e2e/utils.py @@ -0,0 +1,100 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from http import HTTPStatus +from typing import List, Optional, Dict, Union, Callable + +import cortex as cx +import requests +import yaml + + +def wait_for(fn: Callable[[], bool], timeout=None) -> bool: + deadline = time.time() + timeout if timeout else None + while True: + if deadline is not None and time.time() > deadline: + return False + + done = fn() + if done: + return True + + time.sleep(1) + + +def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = None) -> bool: + def _is_ready(): + return all( + [client.get_api(name)["status"]["status_code"] == "status_live" for name in api_names] + ) + + return wait_for(_is_ready, timeout=timeout) + + +def endpoint_ready(client: cx.Client, api_name: str, timeout: int = None) -> bool: + def _is_ready(): + endpoint = client.get_api(api_name)["endpoint"] + response = requests.post(endpoint) + return response.status_code == HTTPStatus.BAD_REQUEST + + return wait_for(_is_ready, timeout=timeout) + + +def job_done(client: cx.Client, api_name: str, job_id: str, timeout: int = None): + def _is_ready(): + job_info = client.get_job(api_name, job_id) + return job_info["job_status"]["status"] == "status_succeeded" + + return wait_for(_is_ready, timeout=timeout) + + +def request_prediction( + client: cx.Client, api_name: str, payload: Union[List, Dict] +) -> requests.Response: + api_info = client.get_api(api_name) + response = requests.post(api_info["endpoint"], json=payload) + + return response + + +def request_batch_prediction( + client: cx.Client, + api_name: str, + item_list: List, + batch_size: int, + workers: int = 1, + config: Dict = None, +) -> requests.Response: + api_info = client.get_api(api_name) + endpoint = api_info["endpoint"] + + batch_payload = { + "workers": workers, + "item_list": {"items": item_list, "batch_size": batch_size}, + "config": config, + } + response = requests.post(endpoint, json=batch_payload) + + return response + + +def client_from_config(config_path: str) -> cx.Client: + with open(config_path) as f: + config = yaml.safe_load(f) + + cluster_name = config["cluster_name"] + provider = config["provider"] + + return cx.client(f"{cluster_name}-{provider}") diff --git a/test/e2e/pytest.ini b/test/e2e/pytest.ini new file mode 100644 index 0000000000..6b48448ef0 --- /dev/null +++ b/test/e2e/pytest.ini @@ -0,0 +1,4 @@ +# pytest.ini +[pytest] +minversion = 6.0 +addopts = -s -v -r sxf diff --git a/test/e2e/setup.py b/test/e2e/setup.py new file mode 100644 index 0000000000..4ad6789a2d --- /dev/null +++ b/test/e2e/setup.py @@ -0,0 +1,44 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from setuptools import setup, find_packages + +root = Path(__file__).parent.absolute() +cortex_client_dir = root.parent.parent / "pkg" / "cortex" / "client" + +if not cortex_client_dir.exists(): + raise ModuleNotFoundError(f"cortex client not found in {cortex_client_dir}") + +setup( + name="e2e", + version="master", + packages=find_packages(exclude=["tests"]), + url="https://github.com/cortexlabs/cortex", + license="Apache License 2.0", + python_requires=">=3.6", + install_requires=[ + "requests==2.24.0", + "jsonschema==3.2.0", + "pytest==6.1.*", + "python-dotenv==0.15.0", + "pyyaml>=5.3.1", + "cortex", + ], + dependency_links=[f"file://{cortex_client_dir}#egg=cortex"], + author="Cortex Labs", + author_email="dev@cortex.dev", + description="Cortex E2E tests package", +) diff --git a/test/e2e/tests/__init__.py b/test/e2e/tests/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/e2e/tests/aws/__init__.py b/test/e2e/tests/aws/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/aws/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/e2e/tests/aws/conftest.py b/test/e2e/tests/aws/conftest.py new file mode 100644 index 0000000000..90a7c65744 --- /dev/null +++ b/test/e2e/tests/aws/conftest.py @@ -0,0 +1,44 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cortex as cx +import pytest + +import e2e +from e2e.utils import client_from_config + + +@pytest.fixture +def client(config): + env_name = config["aws"]["env"] + if env_name: + return cx.client(env_name) + + config_path = config["aws"]["config"] + if config_path is not None: + return client_from_config(config_path) + + pytest.skip("--aws-env or --aws-config must be passed to run aws tests") + + +def pytest_configure(config): + aws_config = config.getoption("--aws-config") + if aws_config: + e2e.create_cluster(aws_config) + + +def pytest_unconfigure(config): + aws_config = config.getoption("--aws-config") + if aws_config: + e2e.delete_cluster(aws_config) diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py new file mode 100644 index 0000000000..8599f7cf3c --- /dev/null +++ b/test/e2e/tests/aws/test_batch.py @@ -0,0 +1,42 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import cortex as cx +import pytest + +import e2e.tests + +TEST_APIS = ["batch/image-classifier", "batch/onnx", "batch/tensorflow"] + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS) +def test_batch_api(config: Dict, client: cx.Client, api: str): + s3_bucket = config["aws"].get("s3_bucket") + if not s3_bucket: + pytest.skip( + "--s3-bucket option is required to run batch tests (alternatively set the " + "CORTEX_TEST_BATCH_S3_BUCKET_DIR env var) )" + ) + + e2e.tests.test_batch_api( + client, + api, + test_bucket=s3_bucket, + deploy_timeout=config["global"]["batch_deploy_timeout"], + job_timeout=config["global"]["batch_job_timeout"], + retry_attempts=5, + ) diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py new file mode 100644 index 0000000000..f9d1274175 --- /dev/null +++ b/test/e2e/tests/aws/test_realtime.py @@ -0,0 +1,29 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +import cortex as cx +import pytest + +import e2e.tests + +TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS) +def test_realtime_api(config: Dict, client: cx.Client, api: str): + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + ) diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py new file mode 100644 index 0000000000..b8515c93ea --- /dev/null +++ b/test/e2e/tests/conftest.py @@ -0,0 +1,93 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest +import yaml +from dotenv import load_dotenv + + +def pytest_addoption(parser): + parser.addoption( + "--aws-env", + action="store", + default=None, + help="set cortex AWS environment, to test on an existing AWS cluster", + ) + parser.addoption( + "--aws-config", + action="store", + default=None, + help="set cortex AWS cluster config, to test on a new AWS cluster", + ) + parser.addoption( + "--gcp-env", + action="store", + default=None, + help="set cortex GCP environment, to test on an existing GCP cluster", + ) + parser.addoption( + "--gcp-config", + action="store", + default=None, + help="set cortex GCP cluster config, to test on a new GCP cluster", + ) + parser.addoption( + "--s3-bucket", + action="store", + default=None, + help="set s3 bucket where batch jobs results will be stored", + ) + + +def pytest_configure(config): + load_dotenv(".env") + + s3_bucket = os.environ.get("CORTEX_TEST_BATCH_S3_BUCKET_DIR") + s3_bucket = config.getoption("--s3-bucket") if not s3_bucket else s3_bucket + + configuration = { + "aws": { + "env": config.getoption("--aws-env"), + "config": config.getoption("--aws-config"), + "s3_bucket": s3_bucket, + }, + "gcp": { + "env": config.getoption("--gcp-env"), + "config": config.getoption("--gcp-config"), + }, + "global": { + "realtime_deploy_timeout": int( + os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 120) + ), + "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), + "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 120)), + }, + } + + class Config: + @pytest.fixture(autouse=True) + def config(self): + return configuration + + config.pluginmanager.register(Config()) + + print("\n----- Test Configuration -----\n") + print(yaml.dump(configuration, indent=2)) + + if configuration["aws"]["env"] and configuration["aws"]["config"]: + raise ValueError("--aws-env and --aws-config are mutually exclusive") + + if configuration["gcp"]["env"] and configuration["gcp"]["config"]: + raise ValueError("--gcp-env and --gcp-config are mutually exclusive") diff --git a/test/e2e/tests/gcp/__init__.py b/test/e2e/tests/gcp/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/gcp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py new file mode 100644 index 0000000000..58fde4be81 --- /dev/null +++ b/test/e2e/tests/gcp/conftest.py @@ -0,0 +1,45 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +import cortex as cx +import pytest + +import e2e +from e2e.utils import client_from_config + + +@pytest.fixture +def client(config: Dict): + env_name = config["gcp"]["env"] + if env_name: + return cx.client(env_name) + + config_path = config["gcp"]["config"] + if config_path is not None: + return client_from_config(config_path) + + pytest.skip("--gcp-env or --gcp-config must be passed to run gcp tests") + + +def pytest_configure(config): + gcp_config = config.getoption("--gcp-config") + if gcp_config: + e2e.create_cluster(gcp_config) + + +def pytest_unconfigure(config): + gcp_config = config.getoption("--gcp-config") + if gcp_config: + e2e.delete_cluster(gcp_config) diff --git a/test/e2e/tests/gcp/test_realtime.py b/test/e2e/tests/gcp/test_realtime.py new file mode 100644 index 0000000000..4062ca30ed --- /dev/null +++ b/test/e2e/tests/gcp/test_realtime.py @@ -0,0 +1,29 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict + +import cortex as cx +import pytest + +import e2e.tests + +TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS) +def test_realtime_apis(config: Dict, client: cx.Client, api: str): + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + )