From 879f52666db4e96868196eb4beda2154ec13af04 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 10 Dec 2020 14:30:42 +0100 Subject: [PATCH 01/31] Move test apis to an api/ folder --- test/{ => apis}/batch/image-classifier/README.md | 0 test/{ => apis}/batch/image-classifier/cortex.yaml | 0 test/{ => apis}/batch/image-classifier/predictor.py | 0 .../batch/image-classifier/requirements.txt | 0 test/{ => apis}/batch/image-classifier/sample.json | 0 test/{ => apis}/batch/onnx/cortex.yaml | 0 test/{ => apis}/batch/onnx/predictor.py | 0 test/{ => apis}/batch/onnx/requirements.txt | 0 test/{ => apis}/batch/tensorflow/cortex.yaml | 0 test/{ => apis}/batch/tensorflow/predictor.py | 0 test/{ => apis}/batch/tensorflow/requirements.txt | 0 test/{ => apis}/keras/document-denoiser/README.md | 0 test/{ => apis}/keras/document-denoiser/cortex.yaml | 0 .../{ => apis}/keras/document-denoiser/predictor.py | 0 .../keras/document-denoiser/requirements.txt | 0 test/{ => apis}/keras/document-denoiser/sample.json | 0 .../keras/document-denoiser/trainer.ipynb | 0 test/{ => apis}/live-reloading/onnx/README.md | 0 .../live-reloading/python/mpg-estimator/cortex.yaml | 0 .../python/mpg-estimator/predictor.py | 1 - .../python/mpg-estimator/requirements.txt | 0 .../live-reloading/python/mpg-estimator/sample.json | 0 test/{ => apis}/live-reloading/tensorflow/README.md | 0 .../onnx/multi-model-classifier/README.md | 0 .../onnx/multi-model-classifier/cortex.yaml | 0 .../onnx/multi-model-classifier/predictor.py | 0 .../onnx/multi-model-classifier/requirements.txt | 0 .../onnx/multi-model-classifier/sample.json | 0 .../model-caching/python/mpg-estimator/README.md | 0 .../model-caching/python/mpg-estimator/cortex.yaml | 0 .../model-caching/python/mpg-estimator/predictor.py | 1 - .../python/mpg-estimator/requirements.txt | 0 .../model-caching/python/mpg-estimator/sample.json | 0 .../tensorflow/multi-model-classifier/README.md | 0 .../tensorflow/multi-model-classifier/cortex.yaml | 0 .../tensorflow/multi-model-classifier/predictor.py | 0 .../multi-model-classifier/requirements.txt | 0 .../multi-model-classifier/sample-image.json | 0 .../multi-model-classifier/sample-iris.json | 0 test/{ => apis}/onnx/iris-classifier/cortex.yaml | 0 test/{ => apis}/onnx/iris-classifier/predictor.py | 0 test/{ => apis}/onnx/iris-classifier/sample.json | 0 test/{ => apis}/onnx/iris-classifier/xgboost.ipynb | 0 .../onnx/multi-model-classifier/README.md | 0 .../onnx/multi-model-classifier/cortex.yaml | 0 .../onnx/multi-model-classifier/predictor.py | 0 .../onnx/multi-model-classifier/requirements.txt | 0 .../onnx/multi-model-classifier/sample.json | 0 test/{ => apis}/onnx/yolov5-youtube/README.md | 0 .../onnx/yolov5-youtube/conda-packages.txt | 0 test/{ => apis}/onnx/yolov5-youtube/cortex.yaml | 0 test/{ => apis}/onnx/yolov5-youtube/labels.json | 0 test/{ => apis}/onnx/yolov5-youtube/predictor.py | 0 .../{ => apis}/onnx/yolov5-youtube/requirements.txt | 0 test/{ => apis}/onnx/yolov5-youtube/sample.json | 0 test/{ => apis}/onnx/yolov5-youtube/utils.py | 0 .../{ => apis}/pytorch/answer-generator/cortex.yaml | 0 .../pytorch/answer-generator/generator.py | 0 .../pytorch/answer-generator/predictor.py | 0 .../pytorch/answer-generator/requirements.txt | 0 .../{ => apis}/pytorch/answer-generator/sample.json | 0 .../pytorch/image-classifier-alexnet/cortex.yaml | 0 .../pytorch/image-classifier-alexnet/predictor.py | 0 .../image-classifier-alexnet/requirements.txt | 0 .../pytorch/image-classifier-alexnet/sample.json | 0 .../pytorch/image-classifier-resnet50/README.md | 0 .../pytorch/image-classifier-resnet50/cortex.yaml | 0 .../image-classifier-resnet50/cortex_gpu.yaml | 0 .../image-classifier-resnet50/cortex_inf.yaml | 0 .../generate_resnet50_models.ipynb | 0 .../pytorch/image-classifier-resnet50/predictor.py | 0 .../pytorch/image-classifier-resnet50/sample.json | 0 test/{ => apis}/pytorch/iris-classifier/cortex.yaml | 0 test/apis/pytorch/iris-classifier/expectations.yaml | 5 +++++ test/{ => apis}/pytorch/iris-classifier/model.py | 0 .../{ => apis}/pytorch/iris-classifier/predictor.py | 0 .../pytorch/iris-classifier/requirements.txt | 0 test/{ => apis}/pytorch/iris-classifier/sample.json | 0 .../pytorch/language-identifier/cortex.yaml | 0 .../pytorch/language-identifier/predictor.py | 0 .../pytorch/language-identifier/requirements.txt | 0 .../pytorch/language-identifier/sample.json | 0 .../pytorch/multi-model-text-analyzer/README.md | 0 .../pytorch/multi-model-text-analyzer/cortex.yaml | 0 .../pytorch/multi-model-text-analyzer/predictor.py | 0 .../multi-model-text-analyzer/requirements.txt | 0 .../multi-model-text-analyzer/sample-sentiment.json | 0 .../sample-summarizer.json | 0 .../pytorch/object-detector/coco_labels.txt | 0 test/{ => apis}/pytorch/object-detector/cortex.yaml | 0 .../{ => apis}/pytorch/object-detector/predictor.py | 0 .../pytorch/object-detector/requirements.txt | 0 test/{ => apis}/pytorch/object-detector/sample.json | 0 .../pytorch/question-generator/cortex.yaml | 0 .../pytorch/question-generator/dependencies.sh | 0 .../pytorch/question-generator/predictor.py | 0 .../pytorch/question-generator/requirements.txt | 0 .../pytorch/question-generator/sample.json | 0 .../pytorch/reading-comprehender/cortex.yaml | 0 .../pytorch/reading-comprehender/predictor.py | 0 .../pytorch/reading-comprehender/requirements.txt | 0 .../pytorch/reading-comprehender/sample.json | 0 .../{ => apis}/pytorch/search-completer/cortex.yaml | 0 .../pytorch/search-completer/predictor.py | 0 .../pytorch/search-completer/requirements.txt | 0 .../{ => apis}/pytorch/search-completer/sample.json | 0 .../pytorch/sentiment-analyzer/cortex.yaml | 0 .../pytorch/sentiment-analyzer/predictor.py | 0 .../pytorch/sentiment-analyzer/requirements.txt | 0 .../pytorch/sentiment-analyzer/sample.json | 0 test/{ => apis}/pytorch/text-generator/README.md | 0 test/{ => apis}/pytorch/text-generator/cortex.yaml | 0 test/{ => apis}/pytorch/text-generator/deploy.ipynb | 0 test/{ => apis}/pytorch/text-generator/predictor.py | 0 .../pytorch/text-generator/requirements.txt | 0 test/{ => apis}/pytorch/text-generator/sample.json | 0 test/{ => apis}/pytorch/text-summarizer/README.md | 0 test/{ => apis}/pytorch/text-summarizer/cortex.yaml | 0 .../{ => apis}/pytorch/text-summarizer/predictor.py | 0 .../pytorch/text-summarizer/requirements.txt | 0 test/{ => apis}/pytorch/text-summarizer/sample.json | 0 test/{ => apis}/sklearn/iris-classifier/cortex.yaml | 0 .../{ => apis}/sklearn/iris-classifier/predictor.py | 0 .../sklearn/iris-classifier/requirements.txt | 0 test/{ => apis}/sklearn/iris-classifier/sample.json | 0 test/{ => apis}/sklearn/iris-classifier/trainer.py | 0 test/{ => apis}/sklearn/mpg-estimator/cortex.yaml | 0 test/{ => apis}/sklearn/mpg-estimator/predictor.py | 0 .../sklearn/mpg-estimator/requirements.txt | 0 test/{ => apis}/sklearn/mpg-estimator/sample.json | 0 test/{ => apis}/sklearn/mpg-estimator/trainer.py | 0 test/{ => apis}/spacy/entity-recognizer/cortex.yaml | 0 .../{ => apis}/spacy/entity-recognizer/predictor.py | 0 .../spacy/entity-recognizer/requirements.txt | 0 test/{ => apis}/spacy/entity-recognizer/sample.json | 0 .../image-classifier-inception/cortex.yaml | 0 .../cortex_server_side_batching.yaml | 0 .../image-classifier-inception/inception.ipynb | 0 .../image-classifier-inception/predictor.py | 0 .../image-classifier-inception/requirements.txt | 0 .../image-classifier-inception/sample.json | 0 .../tensorflow/image-classifier-resnet50/README.md | 0 .../image-classifier-resnet50/cortex.yaml | 0 .../image-classifier-resnet50/cortex_gpu.yaml | 0 .../cortex_gpu_server_side_batching.yaml | 0 .../image-classifier-resnet50/cortex_inf.yaml | 0 .../cortex_inf_server_side_batching.yaml | 0 .../generate_gpu_resnet50_model.ipynb | 0 .../generate_resnet50_models.ipynb | 0 .../image-classifier-resnet50/predictor.py | 0 .../image-classifier-resnet50/requirements.txt | 0 .../tensorflow/image-classifier-resnet50/sample.bin | Bin .../image-classifier-resnet50/sample.json | 0 .../tensorflow/iris-classifier/cortex.yaml | 0 .../tensorflow/iris-classifier/predictor.py | 0 .../tensorflow/iris-classifier/sample.json | 0 .../tensorflow/iris-classifier/tensorflow.ipynb | 0 .../tensorflow/license-plate-reader/README.md | 0 .../tensorflow/license-plate-reader/config.json | 0 .../license-plate-reader/cortex_full.yaml | 0 .../license-plate-reader/cortex_lite.yaml | 0 .../license-plate-reader/predictor_crnn.py | 0 .../license-plate-reader/predictor_lite.py | 0 .../license-plate-reader/predictor_yolo.py | 0 .../license-plate-reader/requirements.txt | 0 .../license-plate-reader/sample_inference.py | 0 .../license-plate-reader/utils/__init__.py | 0 .../tensorflow/license-plate-reader/utils/bbox.py | 0 .../tensorflow/license-plate-reader/utils/colors.py | 0 .../license-plate-reader/utils/preprocess.py | 0 .../tensorflow/license-plate-reader/utils/utils.py | 0 .../tensorflow/multi-model-classifier/README.md | 0 .../tensorflow/multi-model-classifier/cortex.yaml | 0 .../tensorflow/multi-model-classifier/predictor.py | 0 .../multi-model-classifier/requirements.txt | 0 .../multi-model-classifier/sample-image.json | 0 .../multi-model-classifier/sample-iris.json | 0 .../tensorflow/sentiment-analyzer/bert.ipynb | 0 .../tensorflow/sentiment-analyzer/cortex.yaml | 0 .../tensorflow/sentiment-analyzer/predictor.py | 0 .../tensorflow/sentiment-analyzer/requirements.txt | 0 .../tensorflow/sentiment-analyzer/sample.json | 0 .../tensorflow/text-generator/cortex.yaml | 0 .../{ => apis}/tensorflow/text-generator/encoder.py | 0 .../tensorflow/text-generator/gpt-2.ipynb | 0 .../tensorflow/text-generator/predictor.py | 0 .../tensorflow/text-generator/requirements.txt | 0 .../tensorflow/text-generator/sample.json | 0 test/{ => apis}/traffic-splitter/README.md | 0 test/{ => apis}/traffic-splitter/cortex.yaml | 0 test/{ => apis}/traffic-splitter/model.py | 0 test/{ => apis}/traffic-splitter/onnx_predictor.py | 0 .../traffic-splitter/pytorch_predictor.py | 0 test/{ => apis}/traffic-splitter/sample.json | 0 194 files changed, 5 insertions(+), 2 deletions(-) rename test/{ => apis}/batch/image-classifier/README.md (100%) rename test/{ => apis}/batch/image-classifier/cortex.yaml (100%) rename test/{ => apis}/batch/image-classifier/predictor.py (100%) rename test/{ => apis}/batch/image-classifier/requirements.txt (100%) rename test/{ => apis}/batch/image-classifier/sample.json (100%) rename test/{ => apis}/batch/onnx/cortex.yaml (100%) rename test/{ => apis}/batch/onnx/predictor.py (100%) rename test/{ => apis}/batch/onnx/requirements.txt (100%) rename test/{ => apis}/batch/tensorflow/cortex.yaml (100%) rename test/{ => apis}/batch/tensorflow/predictor.py (100%) rename test/{ => apis}/batch/tensorflow/requirements.txt (100%) rename test/{ => apis}/keras/document-denoiser/README.md (100%) rename test/{ => apis}/keras/document-denoiser/cortex.yaml (100%) rename test/{ => apis}/keras/document-denoiser/predictor.py (100%) rename test/{ => apis}/keras/document-denoiser/requirements.txt (100%) rename test/{ => apis}/keras/document-denoiser/sample.json (100%) rename test/{ => apis}/keras/document-denoiser/trainer.ipynb (100%) rename test/{ => apis}/live-reloading/onnx/README.md (100%) rename test/{ => apis}/live-reloading/python/mpg-estimator/cortex.yaml (100%) rename test/{ => apis}/live-reloading/python/mpg-estimator/predictor.py (97%) rename test/{ => apis}/live-reloading/python/mpg-estimator/requirements.txt (100%) rename test/{ => apis}/live-reloading/python/mpg-estimator/sample.json (100%) rename test/{ => apis}/live-reloading/tensorflow/README.md (100%) rename test/{ => apis}/model-caching/onnx/multi-model-classifier/README.md (100%) rename test/{ => apis}/model-caching/onnx/multi-model-classifier/cortex.yaml (100%) rename test/{ => apis}/model-caching/onnx/multi-model-classifier/predictor.py (100%) rename test/{ => apis}/model-caching/onnx/multi-model-classifier/requirements.txt (100%) rename test/{ => apis}/model-caching/onnx/multi-model-classifier/sample.json (100%) rename test/{ => apis}/model-caching/python/mpg-estimator/README.md (100%) rename test/{ => apis}/model-caching/python/mpg-estimator/cortex.yaml (100%) rename test/{ => apis}/model-caching/python/mpg-estimator/predictor.py (97%) rename test/{ => apis}/model-caching/python/mpg-estimator/requirements.txt (100%) rename test/{ => apis}/model-caching/python/mpg-estimator/sample.json (100%) rename test/{ => apis}/model-caching/tensorflow/multi-model-classifier/README.md (100%) rename test/{ => apis}/model-caching/tensorflow/multi-model-classifier/cortex.yaml (100%) rename test/{ => apis}/model-caching/tensorflow/multi-model-classifier/predictor.py (100%) rename test/{ => apis}/model-caching/tensorflow/multi-model-classifier/requirements.txt (100%) rename test/{ => apis}/model-caching/tensorflow/multi-model-classifier/sample-image.json (100%) rename test/{ => apis}/model-caching/tensorflow/multi-model-classifier/sample-iris.json (100%) rename test/{ => apis}/onnx/iris-classifier/cortex.yaml (100%) rename test/{ => apis}/onnx/iris-classifier/predictor.py (100%) rename test/{ => apis}/onnx/iris-classifier/sample.json (100%) rename test/{ => apis}/onnx/iris-classifier/xgboost.ipynb (100%) rename test/{ => apis}/onnx/multi-model-classifier/README.md (100%) rename test/{ => apis}/onnx/multi-model-classifier/cortex.yaml (100%) rename test/{ => apis}/onnx/multi-model-classifier/predictor.py (100%) rename test/{ => apis}/onnx/multi-model-classifier/requirements.txt (100%) rename test/{ => apis}/onnx/multi-model-classifier/sample.json (100%) rename test/{ => apis}/onnx/yolov5-youtube/README.md (100%) rename test/{ => apis}/onnx/yolov5-youtube/conda-packages.txt (100%) rename test/{ => apis}/onnx/yolov5-youtube/cortex.yaml (100%) rename test/{ => apis}/onnx/yolov5-youtube/labels.json (100%) rename test/{ => apis}/onnx/yolov5-youtube/predictor.py (100%) rename test/{ => apis}/onnx/yolov5-youtube/requirements.txt (100%) rename test/{ => apis}/onnx/yolov5-youtube/sample.json (100%) rename test/{ => apis}/onnx/yolov5-youtube/utils.py (100%) rename test/{ => apis}/pytorch/answer-generator/cortex.yaml (100%) rename test/{ => apis}/pytorch/answer-generator/generator.py (100%) rename test/{ => apis}/pytorch/answer-generator/predictor.py (100%) rename test/{ => apis}/pytorch/answer-generator/requirements.txt (100%) rename test/{ => apis}/pytorch/answer-generator/sample.json (100%) rename test/{ => apis}/pytorch/image-classifier-alexnet/cortex.yaml (100%) rename test/{ => apis}/pytorch/image-classifier-alexnet/predictor.py (100%) rename test/{ => apis}/pytorch/image-classifier-alexnet/requirements.txt (100%) rename test/{ => apis}/pytorch/image-classifier-alexnet/sample.json (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/README.md (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/cortex.yaml (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/cortex_gpu.yaml (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/cortex_inf.yaml (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/predictor.py (100%) rename test/{ => apis}/pytorch/image-classifier-resnet50/sample.json (100%) rename test/{ => apis}/pytorch/iris-classifier/cortex.yaml (100%) create mode 100644 test/apis/pytorch/iris-classifier/expectations.yaml rename test/{ => apis}/pytorch/iris-classifier/model.py (100%) rename test/{ => apis}/pytorch/iris-classifier/predictor.py (100%) rename test/{ => apis}/pytorch/iris-classifier/requirements.txt (100%) rename test/{ => apis}/pytorch/iris-classifier/sample.json (100%) rename test/{ => apis}/pytorch/language-identifier/cortex.yaml (100%) rename test/{ => apis}/pytorch/language-identifier/predictor.py (100%) rename test/{ => apis}/pytorch/language-identifier/requirements.txt (100%) rename test/{ => apis}/pytorch/language-identifier/sample.json (100%) rename test/{ => apis}/pytorch/multi-model-text-analyzer/README.md (100%) rename test/{ => apis}/pytorch/multi-model-text-analyzer/cortex.yaml (100%) rename test/{ => apis}/pytorch/multi-model-text-analyzer/predictor.py (100%) rename test/{ => apis}/pytorch/multi-model-text-analyzer/requirements.txt (100%) rename test/{ => apis}/pytorch/multi-model-text-analyzer/sample-sentiment.json (100%) rename test/{ => apis}/pytorch/multi-model-text-analyzer/sample-summarizer.json (100%) rename test/{ => apis}/pytorch/object-detector/coco_labels.txt (100%) rename test/{ => apis}/pytorch/object-detector/cortex.yaml (100%) rename test/{ => apis}/pytorch/object-detector/predictor.py (100%) rename test/{ => apis}/pytorch/object-detector/requirements.txt (100%) rename test/{ => apis}/pytorch/object-detector/sample.json (100%) rename test/{ => apis}/pytorch/question-generator/cortex.yaml (100%) rename test/{ => apis}/pytorch/question-generator/dependencies.sh (100%) rename test/{ => apis}/pytorch/question-generator/predictor.py (100%) rename test/{ => apis}/pytorch/question-generator/requirements.txt (100%) rename test/{ => apis}/pytorch/question-generator/sample.json (100%) rename test/{ => apis}/pytorch/reading-comprehender/cortex.yaml (100%) rename test/{ => apis}/pytorch/reading-comprehender/predictor.py (100%) rename test/{ => apis}/pytorch/reading-comprehender/requirements.txt (100%) rename test/{ => apis}/pytorch/reading-comprehender/sample.json (100%) rename test/{ => apis}/pytorch/search-completer/cortex.yaml (100%) rename test/{ => apis}/pytorch/search-completer/predictor.py (100%) rename test/{ => apis}/pytorch/search-completer/requirements.txt (100%) rename test/{ => apis}/pytorch/search-completer/sample.json (100%) rename test/{ => apis}/pytorch/sentiment-analyzer/cortex.yaml (100%) rename test/{ => apis}/pytorch/sentiment-analyzer/predictor.py (100%) rename test/{ => apis}/pytorch/sentiment-analyzer/requirements.txt (100%) rename test/{ => apis}/pytorch/sentiment-analyzer/sample.json (100%) rename test/{ => apis}/pytorch/text-generator/README.md (100%) rename test/{ => apis}/pytorch/text-generator/cortex.yaml (100%) rename test/{ => apis}/pytorch/text-generator/deploy.ipynb (100%) rename test/{ => apis}/pytorch/text-generator/predictor.py (100%) rename test/{ => apis}/pytorch/text-generator/requirements.txt (100%) rename test/{ => apis}/pytorch/text-generator/sample.json (100%) rename test/{ => apis}/pytorch/text-summarizer/README.md (100%) rename test/{ => apis}/pytorch/text-summarizer/cortex.yaml (100%) rename test/{ => apis}/pytorch/text-summarizer/predictor.py (100%) rename test/{ => apis}/pytorch/text-summarizer/requirements.txt (100%) rename test/{ => apis}/pytorch/text-summarizer/sample.json (100%) rename test/{ => apis}/sklearn/iris-classifier/cortex.yaml (100%) rename test/{ => apis}/sklearn/iris-classifier/predictor.py (100%) rename test/{ => apis}/sklearn/iris-classifier/requirements.txt (100%) rename test/{ => apis}/sklearn/iris-classifier/sample.json (100%) rename test/{ => apis}/sklearn/iris-classifier/trainer.py (100%) rename test/{ => apis}/sklearn/mpg-estimator/cortex.yaml (100%) rename test/{ => apis}/sklearn/mpg-estimator/predictor.py (100%) rename test/{ => apis}/sklearn/mpg-estimator/requirements.txt (100%) rename test/{ => apis}/sklearn/mpg-estimator/sample.json (100%) rename test/{ => apis}/sklearn/mpg-estimator/trainer.py (100%) rename test/{ => apis}/spacy/entity-recognizer/cortex.yaml (100%) rename test/{ => apis}/spacy/entity-recognizer/predictor.py (100%) rename test/{ => apis}/spacy/entity-recognizer/requirements.txt (100%) rename test/{ => apis}/spacy/entity-recognizer/sample.json (100%) rename test/{ => apis}/tensorflow/image-classifier-inception/cortex.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-inception/inception.ipynb (100%) rename test/{ => apis}/tensorflow/image-classifier-inception/predictor.py (100%) rename test/{ => apis}/tensorflow/image-classifier-inception/requirements.txt (100%) rename test/{ => apis}/tensorflow/image-classifier-inception/sample.json (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/README.md (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/cortex.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/cortex_gpu.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/cortex_inf.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/predictor.py (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/requirements.txt (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/sample.bin (100%) rename test/{ => apis}/tensorflow/image-classifier-resnet50/sample.json (100%) rename test/{ => apis}/tensorflow/iris-classifier/cortex.yaml (100%) rename test/{ => apis}/tensorflow/iris-classifier/predictor.py (100%) rename test/{ => apis}/tensorflow/iris-classifier/sample.json (100%) rename test/{ => apis}/tensorflow/iris-classifier/tensorflow.ipynb (100%) rename test/{ => apis}/tensorflow/license-plate-reader/README.md (100%) rename test/{ => apis}/tensorflow/license-plate-reader/config.json (100%) rename test/{ => apis}/tensorflow/license-plate-reader/cortex_full.yaml (100%) rename test/{ => apis}/tensorflow/license-plate-reader/cortex_lite.yaml (100%) rename test/{ => apis}/tensorflow/license-plate-reader/predictor_crnn.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/predictor_lite.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/predictor_yolo.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/requirements.txt (100%) rename test/{ => apis}/tensorflow/license-plate-reader/sample_inference.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/utils/__init__.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/utils/bbox.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/utils/colors.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/utils/preprocess.py (100%) rename test/{ => apis}/tensorflow/license-plate-reader/utils/utils.py (100%) rename test/{ => apis}/tensorflow/multi-model-classifier/README.md (100%) rename test/{ => apis}/tensorflow/multi-model-classifier/cortex.yaml (100%) rename test/{ => apis}/tensorflow/multi-model-classifier/predictor.py (100%) rename test/{ => apis}/tensorflow/multi-model-classifier/requirements.txt (100%) rename test/{ => apis}/tensorflow/multi-model-classifier/sample-image.json (100%) rename test/{ => apis}/tensorflow/multi-model-classifier/sample-iris.json (100%) rename test/{ => apis}/tensorflow/sentiment-analyzer/bert.ipynb (100%) rename test/{ => apis}/tensorflow/sentiment-analyzer/cortex.yaml (100%) rename test/{ => apis}/tensorflow/sentiment-analyzer/predictor.py (100%) rename test/{ => apis}/tensorflow/sentiment-analyzer/requirements.txt (100%) rename test/{ => apis}/tensorflow/sentiment-analyzer/sample.json (100%) rename test/{ => apis}/tensorflow/text-generator/cortex.yaml (100%) rename test/{ => apis}/tensorflow/text-generator/encoder.py (100%) rename test/{ => apis}/tensorflow/text-generator/gpt-2.ipynb (100%) rename test/{ => apis}/tensorflow/text-generator/predictor.py (100%) rename test/{ => apis}/tensorflow/text-generator/requirements.txt (100%) rename test/{ => apis}/tensorflow/text-generator/sample.json (100%) rename test/{ => apis}/traffic-splitter/README.md (100%) rename test/{ => apis}/traffic-splitter/cortex.yaml (100%) rename test/{ => apis}/traffic-splitter/model.py (100%) rename test/{ => apis}/traffic-splitter/onnx_predictor.py (100%) rename test/{ => apis}/traffic-splitter/pytorch_predictor.py (100%) rename test/{ => apis}/traffic-splitter/sample.json (100%) diff --git a/test/batch/image-classifier/README.md b/test/apis/batch/image-classifier/README.md similarity index 100% rename from test/batch/image-classifier/README.md rename to test/apis/batch/image-classifier/README.md diff --git a/test/batch/image-classifier/cortex.yaml b/test/apis/batch/image-classifier/cortex.yaml similarity index 100% rename from test/batch/image-classifier/cortex.yaml rename to test/apis/batch/image-classifier/cortex.yaml diff --git a/test/batch/image-classifier/predictor.py b/test/apis/batch/image-classifier/predictor.py similarity index 100% rename from test/batch/image-classifier/predictor.py rename to test/apis/batch/image-classifier/predictor.py diff --git a/test/batch/image-classifier/requirements.txt b/test/apis/batch/image-classifier/requirements.txt similarity index 100% rename from test/batch/image-classifier/requirements.txt rename to test/apis/batch/image-classifier/requirements.txt diff --git a/test/batch/image-classifier/sample.json b/test/apis/batch/image-classifier/sample.json similarity index 100% rename from test/batch/image-classifier/sample.json rename to test/apis/batch/image-classifier/sample.json diff --git a/test/batch/onnx/cortex.yaml b/test/apis/batch/onnx/cortex.yaml similarity index 100% rename from test/batch/onnx/cortex.yaml rename to test/apis/batch/onnx/cortex.yaml diff --git a/test/batch/onnx/predictor.py b/test/apis/batch/onnx/predictor.py similarity index 100% rename from test/batch/onnx/predictor.py rename to test/apis/batch/onnx/predictor.py diff --git a/test/batch/onnx/requirements.txt b/test/apis/batch/onnx/requirements.txt similarity index 100% rename from test/batch/onnx/requirements.txt rename to test/apis/batch/onnx/requirements.txt diff --git a/test/batch/tensorflow/cortex.yaml b/test/apis/batch/tensorflow/cortex.yaml similarity index 100% rename from test/batch/tensorflow/cortex.yaml rename to test/apis/batch/tensorflow/cortex.yaml diff --git a/test/batch/tensorflow/predictor.py b/test/apis/batch/tensorflow/predictor.py similarity index 100% rename from test/batch/tensorflow/predictor.py rename to test/apis/batch/tensorflow/predictor.py diff --git a/test/batch/tensorflow/requirements.txt b/test/apis/batch/tensorflow/requirements.txt similarity index 100% rename from test/batch/tensorflow/requirements.txt rename to test/apis/batch/tensorflow/requirements.txt diff --git a/test/keras/document-denoiser/README.md b/test/apis/keras/document-denoiser/README.md similarity index 100% rename from test/keras/document-denoiser/README.md rename to test/apis/keras/document-denoiser/README.md diff --git a/test/keras/document-denoiser/cortex.yaml b/test/apis/keras/document-denoiser/cortex.yaml similarity index 100% rename from test/keras/document-denoiser/cortex.yaml rename to test/apis/keras/document-denoiser/cortex.yaml diff --git a/test/keras/document-denoiser/predictor.py b/test/apis/keras/document-denoiser/predictor.py similarity index 100% rename from test/keras/document-denoiser/predictor.py rename to test/apis/keras/document-denoiser/predictor.py diff --git a/test/keras/document-denoiser/requirements.txt b/test/apis/keras/document-denoiser/requirements.txt similarity index 100% rename from test/keras/document-denoiser/requirements.txt rename to test/apis/keras/document-denoiser/requirements.txt diff --git a/test/keras/document-denoiser/sample.json b/test/apis/keras/document-denoiser/sample.json similarity index 100% rename from test/keras/document-denoiser/sample.json rename to test/apis/keras/document-denoiser/sample.json diff --git a/test/keras/document-denoiser/trainer.ipynb b/test/apis/keras/document-denoiser/trainer.ipynb similarity index 100% rename from test/keras/document-denoiser/trainer.ipynb rename to test/apis/keras/document-denoiser/trainer.ipynb diff --git a/test/live-reloading/onnx/README.md b/test/apis/live-reloading/onnx/README.md similarity index 100% rename from test/live-reloading/onnx/README.md rename to test/apis/live-reloading/onnx/README.md diff --git a/test/live-reloading/python/mpg-estimator/cortex.yaml b/test/apis/live-reloading/python/mpg-estimator/cortex.yaml similarity index 100% rename from test/live-reloading/python/mpg-estimator/cortex.yaml rename to test/apis/live-reloading/python/mpg-estimator/cortex.yaml diff --git a/test/live-reloading/python/mpg-estimator/predictor.py b/test/apis/live-reloading/python/mpg-estimator/predictor.py similarity index 97% rename from test/live-reloading/python/mpg-estimator/predictor.py rename to test/apis/live-reloading/python/mpg-estimator/predictor.py index eb643efcd7..122fac51fc 100644 --- a/test/live-reloading/python/mpg-estimator/predictor.py +++ b/test/apis/live-reloading/python/mpg-estimator/predictor.py @@ -1,5 +1,4 @@ import mlflow.sklearn -import numpy as np class PythonPredictor: diff --git a/test/live-reloading/python/mpg-estimator/requirements.txt b/test/apis/live-reloading/python/mpg-estimator/requirements.txt similarity index 100% rename from test/live-reloading/python/mpg-estimator/requirements.txt rename to test/apis/live-reloading/python/mpg-estimator/requirements.txt diff --git a/test/live-reloading/python/mpg-estimator/sample.json b/test/apis/live-reloading/python/mpg-estimator/sample.json similarity index 100% rename from test/live-reloading/python/mpg-estimator/sample.json rename to test/apis/live-reloading/python/mpg-estimator/sample.json diff --git a/test/live-reloading/tensorflow/README.md b/test/apis/live-reloading/tensorflow/README.md similarity index 100% rename from test/live-reloading/tensorflow/README.md rename to test/apis/live-reloading/tensorflow/README.md diff --git a/test/model-caching/onnx/multi-model-classifier/README.md b/test/apis/model-caching/onnx/multi-model-classifier/README.md similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/README.md rename to test/apis/model-caching/onnx/multi-model-classifier/README.md diff --git a/test/model-caching/onnx/multi-model-classifier/cortex.yaml b/test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/cortex.yaml rename to test/apis/model-caching/onnx/multi-model-classifier/cortex.yaml diff --git a/test/model-caching/onnx/multi-model-classifier/predictor.py b/test/apis/model-caching/onnx/multi-model-classifier/predictor.py similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/predictor.py rename to test/apis/model-caching/onnx/multi-model-classifier/predictor.py diff --git a/test/model-caching/onnx/multi-model-classifier/requirements.txt b/test/apis/model-caching/onnx/multi-model-classifier/requirements.txt similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/requirements.txt rename to test/apis/model-caching/onnx/multi-model-classifier/requirements.txt diff --git a/test/model-caching/onnx/multi-model-classifier/sample.json b/test/apis/model-caching/onnx/multi-model-classifier/sample.json similarity index 100% rename from test/model-caching/onnx/multi-model-classifier/sample.json rename to test/apis/model-caching/onnx/multi-model-classifier/sample.json diff --git a/test/model-caching/python/mpg-estimator/README.md b/test/apis/model-caching/python/mpg-estimator/README.md similarity index 100% rename from test/model-caching/python/mpg-estimator/README.md rename to test/apis/model-caching/python/mpg-estimator/README.md diff --git a/test/model-caching/python/mpg-estimator/cortex.yaml b/test/apis/model-caching/python/mpg-estimator/cortex.yaml similarity index 100% rename from test/model-caching/python/mpg-estimator/cortex.yaml rename to test/apis/model-caching/python/mpg-estimator/cortex.yaml diff --git a/test/model-caching/python/mpg-estimator/predictor.py b/test/apis/model-caching/python/mpg-estimator/predictor.py similarity index 97% rename from test/model-caching/python/mpg-estimator/predictor.py rename to test/apis/model-caching/python/mpg-estimator/predictor.py index 8fcdb50a5d..146bab6c90 100644 --- a/test/model-caching/python/mpg-estimator/predictor.py +++ b/test/apis/model-caching/python/mpg-estimator/predictor.py @@ -1,5 +1,4 @@ import mlflow.sklearn -import numpy as np class PythonPredictor: diff --git a/test/model-caching/python/mpg-estimator/requirements.txt b/test/apis/model-caching/python/mpg-estimator/requirements.txt similarity index 100% rename from test/model-caching/python/mpg-estimator/requirements.txt rename to test/apis/model-caching/python/mpg-estimator/requirements.txt diff --git a/test/model-caching/python/mpg-estimator/sample.json b/test/apis/model-caching/python/mpg-estimator/sample.json similarity index 100% rename from test/model-caching/python/mpg-estimator/sample.json rename to test/apis/model-caching/python/mpg-estimator/sample.json diff --git a/test/model-caching/tensorflow/multi-model-classifier/README.md b/test/apis/model-caching/tensorflow/multi-model-classifier/README.md similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/README.md rename to test/apis/model-caching/tensorflow/multi-model-classifier/README.md diff --git a/test/model-caching/tensorflow/multi-model-classifier/cortex.yaml b/test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/cortex.yaml rename to test/apis/model-caching/tensorflow/multi-model-classifier/cortex.yaml diff --git a/test/model-caching/tensorflow/multi-model-classifier/predictor.py b/test/apis/model-caching/tensorflow/multi-model-classifier/predictor.py similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/predictor.py rename to test/apis/model-caching/tensorflow/multi-model-classifier/predictor.py diff --git a/test/model-caching/tensorflow/multi-model-classifier/requirements.txt b/test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/requirements.txt rename to test/apis/model-caching/tensorflow/multi-model-classifier/requirements.txt diff --git a/test/model-caching/tensorflow/multi-model-classifier/sample-image.json b/test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/sample-image.json rename to test/apis/model-caching/tensorflow/multi-model-classifier/sample-image.json diff --git a/test/model-caching/tensorflow/multi-model-classifier/sample-iris.json b/test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json similarity index 100% rename from test/model-caching/tensorflow/multi-model-classifier/sample-iris.json rename to test/apis/model-caching/tensorflow/multi-model-classifier/sample-iris.json diff --git a/test/onnx/iris-classifier/cortex.yaml b/test/apis/onnx/iris-classifier/cortex.yaml similarity index 100% rename from test/onnx/iris-classifier/cortex.yaml rename to test/apis/onnx/iris-classifier/cortex.yaml diff --git a/test/onnx/iris-classifier/predictor.py b/test/apis/onnx/iris-classifier/predictor.py similarity index 100% rename from test/onnx/iris-classifier/predictor.py rename to test/apis/onnx/iris-classifier/predictor.py diff --git a/test/onnx/iris-classifier/sample.json b/test/apis/onnx/iris-classifier/sample.json similarity index 100% rename from test/onnx/iris-classifier/sample.json rename to test/apis/onnx/iris-classifier/sample.json diff --git a/test/onnx/iris-classifier/xgboost.ipynb b/test/apis/onnx/iris-classifier/xgboost.ipynb similarity index 100% rename from test/onnx/iris-classifier/xgboost.ipynb rename to test/apis/onnx/iris-classifier/xgboost.ipynb diff --git a/test/onnx/multi-model-classifier/README.md b/test/apis/onnx/multi-model-classifier/README.md similarity index 100% rename from test/onnx/multi-model-classifier/README.md rename to test/apis/onnx/multi-model-classifier/README.md diff --git a/test/onnx/multi-model-classifier/cortex.yaml b/test/apis/onnx/multi-model-classifier/cortex.yaml similarity index 100% rename from test/onnx/multi-model-classifier/cortex.yaml rename to test/apis/onnx/multi-model-classifier/cortex.yaml diff --git a/test/onnx/multi-model-classifier/predictor.py b/test/apis/onnx/multi-model-classifier/predictor.py similarity index 100% rename from test/onnx/multi-model-classifier/predictor.py rename to test/apis/onnx/multi-model-classifier/predictor.py diff --git a/test/onnx/multi-model-classifier/requirements.txt b/test/apis/onnx/multi-model-classifier/requirements.txt similarity index 100% rename from test/onnx/multi-model-classifier/requirements.txt rename to test/apis/onnx/multi-model-classifier/requirements.txt diff --git a/test/onnx/multi-model-classifier/sample.json b/test/apis/onnx/multi-model-classifier/sample.json similarity index 100% rename from test/onnx/multi-model-classifier/sample.json rename to test/apis/onnx/multi-model-classifier/sample.json diff --git a/test/onnx/yolov5-youtube/README.md b/test/apis/onnx/yolov5-youtube/README.md similarity index 100% rename from test/onnx/yolov5-youtube/README.md rename to test/apis/onnx/yolov5-youtube/README.md diff --git a/test/onnx/yolov5-youtube/conda-packages.txt b/test/apis/onnx/yolov5-youtube/conda-packages.txt similarity index 100% rename from test/onnx/yolov5-youtube/conda-packages.txt rename to test/apis/onnx/yolov5-youtube/conda-packages.txt diff --git a/test/onnx/yolov5-youtube/cortex.yaml b/test/apis/onnx/yolov5-youtube/cortex.yaml similarity index 100% rename from test/onnx/yolov5-youtube/cortex.yaml rename to test/apis/onnx/yolov5-youtube/cortex.yaml diff --git a/test/onnx/yolov5-youtube/labels.json b/test/apis/onnx/yolov5-youtube/labels.json similarity index 100% rename from test/onnx/yolov5-youtube/labels.json rename to test/apis/onnx/yolov5-youtube/labels.json diff --git a/test/onnx/yolov5-youtube/predictor.py b/test/apis/onnx/yolov5-youtube/predictor.py similarity index 100% rename from test/onnx/yolov5-youtube/predictor.py rename to test/apis/onnx/yolov5-youtube/predictor.py diff --git a/test/onnx/yolov5-youtube/requirements.txt b/test/apis/onnx/yolov5-youtube/requirements.txt similarity index 100% rename from test/onnx/yolov5-youtube/requirements.txt rename to test/apis/onnx/yolov5-youtube/requirements.txt diff --git a/test/onnx/yolov5-youtube/sample.json b/test/apis/onnx/yolov5-youtube/sample.json similarity index 100% rename from test/onnx/yolov5-youtube/sample.json rename to test/apis/onnx/yolov5-youtube/sample.json diff --git a/test/onnx/yolov5-youtube/utils.py b/test/apis/onnx/yolov5-youtube/utils.py similarity index 100% rename from test/onnx/yolov5-youtube/utils.py rename to test/apis/onnx/yolov5-youtube/utils.py diff --git a/test/pytorch/answer-generator/cortex.yaml b/test/apis/pytorch/answer-generator/cortex.yaml similarity index 100% rename from test/pytorch/answer-generator/cortex.yaml rename to test/apis/pytorch/answer-generator/cortex.yaml diff --git a/test/pytorch/answer-generator/generator.py b/test/apis/pytorch/answer-generator/generator.py similarity index 100% rename from test/pytorch/answer-generator/generator.py rename to test/apis/pytorch/answer-generator/generator.py diff --git a/test/pytorch/answer-generator/predictor.py b/test/apis/pytorch/answer-generator/predictor.py similarity index 100% rename from test/pytorch/answer-generator/predictor.py rename to test/apis/pytorch/answer-generator/predictor.py diff --git a/test/pytorch/answer-generator/requirements.txt b/test/apis/pytorch/answer-generator/requirements.txt similarity index 100% rename from test/pytorch/answer-generator/requirements.txt rename to test/apis/pytorch/answer-generator/requirements.txt diff --git a/test/pytorch/answer-generator/sample.json b/test/apis/pytorch/answer-generator/sample.json similarity index 100% rename from test/pytorch/answer-generator/sample.json rename to test/apis/pytorch/answer-generator/sample.json diff --git a/test/pytorch/image-classifier-alexnet/cortex.yaml b/test/apis/pytorch/image-classifier-alexnet/cortex.yaml similarity index 100% rename from test/pytorch/image-classifier-alexnet/cortex.yaml rename to test/apis/pytorch/image-classifier-alexnet/cortex.yaml diff --git a/test/pytorch/image-classifier-alexnet/predictor.py b/test/apis/pytorch/image-classifier-alexnet/predictor.py similarity index 100% rename from test/pytorch/image-classifier-alexnet/predictor.py rename to test/apis/pytorch/image-classifier-alexnet/predictor.py diff --git a/test/pytorch/image-classifier-alexnet/requirements.txt b/test/apis/pytorch/image-classifier-alexnet/requirements.txt similarity index 100% rename from test/pytorch/image-classifier-alexnet/requirements.txt rename to test/apis/pytorch/image-classifier-alexnet/requirements.txt diff --git a/test/pytorch/image-classifier-alexnet/sample.json b/test/apis/pytorch/image-classifier-alexnet/sample.json similarity index 100% rename from test/pytorch/image-classifier-alexnet/sample.json rename to test/apis/pytorch/image-classifier-alexnet/sample.json diff --git a/test/pytorch/image-classifier-resnet50/README.md b/test/apis/pytorch/image-classifier-resnet50/README.md similarity index 100% rename from test/pytorch/image-classifier-resnet50/README.md rename to test/apis/pytorch/image-classifier-resnet50/README.md diff --git a/test/pytorch/image-classifier-resnet50/cortex.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex.yaml similarity index 100% rename from test/pytorch/image-classifier-resnet50/cortex.yaml rename to test/apis/pytorch/image-classifier-resnet50/cortex.yaml diff --git a/test/pytorch/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml similarity index 100% rename from test/pytorch/image-classifier-resnet50/cortex_gpu.yaml rename to test/apis/pytorch/image-classifier-resnet50/cortex_gpu.yaml diff --git a/test/pytorch/image-classifier-resnet50/cortex_inf.yaml b/test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml similarity index 100% rename from test/pytorch/image-classifier-resnet50/cortex_inf.yaml rename to test/apis/pytorch/image-classifier-resnet50/cortex_inf.yaml diff --git a/test/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb b/test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb similarity index 100% rename from test/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb rename to test/apis/pytorch/image-classifier-resnet50/generate_resnet50_models.ipynb diff --git a/test/pytorch/image-classifier-resnet50/predictor.py b/test/apis/pytorch/image-classifier-resnet50/predictor.py similarity index 100% rename from test/pytorch/image-classifier-resnet50/predictor.py rename to test/apis/pytorch/image-classifier-resnet50/predictor.py diff --git a/test/pytorch/image-classifier-resnet50/sample.json b/test/apis/pytorch/image-classifier-resnet50/sample.json similarity index 100% rename from test/pytorch/image-classifier-resnet50/sample.json rename to test/apis/pytorch/image-classifier-resnet50/sample.json diff --git a/test/pytorch/iris-classifier/cortex.yaml b/test/apis/pytorch/iris-classifier/cortex.yaml similarity index 100% rename from test/pytorch/iris-classifier/cortex.yaml rename to test/apis/pytorch/iris-classifier/cortex.yaml diff --git a/test/apis/pytorch/iris-classifier/expectations.yaml b/test/apis/pytorch/iris-classifier/expectations.yaml new file mode 100644 index 0000000000..93b58bac83 --- /dev/null +++ b/test/apis/pytorch/iris-classifier/expectations.yaml @@ -0,0 +1,5 @@ +# this file is used for testing purposes only + +response: + content_type: "text" + expected: "versicolor" diff --git a/test/pytorch/iris-classifier/model.py b/test/apis/pytorch/iris-classifier/model.py similarity index 100% rename from test/pytorch/iris-classifier/model.py rename to test/apis/pytorch/iris-classifier/model.py diff --git a/test/pytorch/iris-classifier/predictor.py b/test/apis/pytorch/iris-classifier/predictor.py similarity index 100% rename from test/pytorch/iris-classifier/predictor.py rename to test/apis/pytorch/iris-classifier/predictor.py diff --git a/test/pytorch/iris-classifier/requirements.txt b/test/apis/pytorch/iris-classifier/requirements.txt similarity index 100% rename from test/pytorch/iris-classifier/requirements.txt rename to test/apis/pytorch/iris-classifier/requirements.txt diff --git a/test/pytorch/iris-classifier/sample.json b/test/apis/pytorch/iris-classifier/sample.json similarity index 100% rename from test/pytorch/iris-classifier/sample.json rename to test/apis/pytorch/iris-classifier/sample.json diff --git a/test/pytorch/language-identifier/cortex.yaml b/test/apis/pytorch/language-identifier/cortex.yaml similarity index 100% rename from test/pytorch/language-identifier/cortex.yaml rename to test/apis/pytorch/language-identifier/cortex.yaml diff --git a/test/pytorch/language-identifier/predictor.py b/test/apis/pytorch/language-identifier/predictor.py similarity index 100% rename from test/pytorch/language-identifier/predictor.py rename to test/apis/pytorch/language-identifier/predictor.py diff --git a/test/pytorch/language-identifier/requirements.txt b/test/apis/pytorch/language-identifier/requirements.txt similarity index 100% rename from test/pytorch/language-identifier/requirements.txt rename to test/apis/pytorch/language-identifier/requirements.txt diff --git a/test/pytorch/language-identifier/sample.json b/test/apis/pytorch/language-identifier/sample.json similarity index 100% rename from test/pytorch/language-identifier/sample.json rename to test/apis/pytorch/language-identifier/sample.json diff --git a/test/pytorch/multi-model-text-analyzer/README.md b/test/apis/pytorch/multi-model-text-analyzer/README.md similarity index 100% rename from test/pytorch/multi-model-text-analyzer/README.md rename to test/apis/pytorch/multi-model-text-analyzer/README.md diff --git a/test/pytorch/multi-model-text-analyzer/cortex.yaml b/test/apis/pytorch/multi-model-text-analyzer/cortex.yaml similarity index 100% rename from test/pytorch/multi-model-text-analyzer/cortex.yaml rename to test/apis/pytorch/multi-model-text-analyzer/cortex.yaml diff --git a/test/pytorch/multi-model-text-analyzer/predictor.py b/test/apis/pytorch/multi-model-text-analyzer/predictor.py similarity index 100% rename from test/pytorch/multi-model-text-analyzer/predictor.py rename to test/apis/pytorch/multi-model-text-analyzer/predictor.py diff --git a/test/pytorch/multi-model-text-analyzer/requirements.txt b/test/apis/pytorch/multi-model-text-analyzer/requirements.txt similarity index 100% rename from test/pytorch/multi-model-text-analyzer/requirements.txt rename to test/apis/pytorch/multi-model-text-analyzer/requirements.txt diff --git a/test/pytorch/multi-model-text-analyzer/sample-sentiment.json b/test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json similarity index 100% rename from test/pytorch/multi-model-text-analyzer/sample-sentiment.json rename to test/apis/pytorch/multi-model-text-analyzer/sample-sentiment.json diff --git a/test/pytorch/multi-model-text-analyzer/sample-summarizer.json b/test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json similarity index 100% rename from test/pytorch/multi-model-text-analyzer/sample-summarizer.json rename to test/apis/pytorch/multi-model-text-analyzer/sample-summarizer.json diff --git a/test/pytorch/object-detector/coco_labels.txt b/test/apis/pytorch/object-detector/coco_labels.txt similarity index 100% rename from test/pytorch/object-detector/coco_labels.txt rename to test/apis/pytorch/object-detector/coco_labels.txt diff --git a/test/pytorch/object-detector/cortex.yaml b/test/apis/pytorch/object-detector/cortex.yaml similarity index 100% rename from test/pytorch/object-detector/cortex.yaml rename to test/apis/pytorch/object-detector/cortex.yaml diff --git a/test/pytorch/object-detector/predictor.py b/test/apis/pytorch/object-detector/predictor.py similarity index 100% rename from test/pytorch/object-detector/predictor.py rename to test/apis/pytorch/object-detector/predictor.py diff --git a/test/pytorch/object-detector/requirements.txt b/test/apis/pytorch/object-detector/requirements.txt similarity index 100% rename from test/pytorch/object-detector/requirements.txt rename to test/apis/pytorch/object-detector/requirements.txt diff --git a/test/pytorch/object-detector/sample.json b/test/apis/pytorch/object-detector/sample.json similarity index 100% rename from test/pytorch/object-detector/sample.json rename to test/apis/pytorch/object-detector/sample.json diff --git a/test/pytorch/question-generator/cortex.yaml b/test/apis/pytorch/question-generator/cortex.yaml similarity index 100% rename from test/pytorch/question-generator/cortex.yaml rename to test/apis/pytorch/question-generator/cortex.yaml diff --git a/test/pytorch/question-generator/dependencies.sh b/test/apis/pytorch/question-generator/dependencies.sh similarity index 100% rename from test/pytorch/question-generator/dependencies.sh rename to test/apis/pytorch/question-generator/dependencies.sh diff --git a/test/pytorch/question-generator/predictor.py b/test/apis/pytorch/question-generator/predictor.py similarity index 100% rename from test/pytorch/question-generator/predictor.py rename to test/apis/pytorch/question-generator/predictor.py diff --git a/test/pytorch/question-generator/requirements.txt b/test/apis/pytorch/question-generator/requirements.txt similarity index 100% rename from test/pytorch/question-generator/requirements.txt rename to test/apis/pytorch/question-generator/requirements.txt diff --git a/test/pytorch/question-generator/sample.json b/test/apis/pytorch/question-generator/sample.json similarity index 100% rename from test/pytorch/question-generator/sample.json rename to test/apis/pytorch/question-generator/sample.json diff --git a/test/pytorch/reading-comprehender/cortex.yaml b/test/apis/pytorch/reading-comprehender/cortex.yaml similarity index 100% rename from test/pytorch/reading-comprehender/cortex.yaml rename to test/apis/pytorch/reading-comprehender/cortex.yaml diff --git a/test/pytorch/reading-comprehender/predictor.py b/test/apis/pytorch/reading-comprehender/predictor.py similarity index 100% rename from test/pytorch/reading-comprehender/predictor.py rename to test/apis/pytorch/reading-comprehender/predictor.py diff --git a/test/pytorch/reading-comprehender/requirements.txt b/test/apis/pytorch/reading-comprehender/requirements.txt similarity index 100% rename from test/pytorch/reading-comprehender/requirements.txt rename to test/apis/pytorch/reading-comprehender/requirements.txt diff --git a/test/pytorch/reading-comprehender/sample.json b/test/apis/pytorch/reading-comprehender/sample.json similarity index 100% rename from test/pytorch/reading-comprehender/sample.json rename to test/apis/pytorch/reading-comprehender/sample.json diff --git a/test/pytorch/search-completer/cortex.yaml b/test/apis/pytorch/search-completer/cortex.yaml similarity index 100% rename from test/pytorch/search-completer/cortex.yaml rename to test/apis/pytorch/search-completer/cortex.yaml diff --git a/test/pytorch/search-completer/predictor.py b/test/apis/pytorch/search-completer/predictor.py similarity index 100% rename from test/pytorch/search-completer/predictor.py rename to test/apis/pytorch/search-completer/predictor.py diff --git a/test/pytorch/search-completer/requirements.txt b/test/apis/pytorch/search-completer/requirements.txt similarity index 100% rename from test/pytorch/search-completer/requirements.txt rename to test/apis/pytorch/search-completer/requirements.txt diff --git a/test/pytorch/search-completer/sample.json b/test/apis/pytorch/search-completer/sample.json similarity index 100% rename from test/pytorch/search-completer/sample.json rename to test/apis/pytorch/search-completer/sample.json diff --git a/test/pytorch/sentiment-analyzer/cortex.yaml b/test/apis/pytorch/sentiment-analyzer/cortex.yaml similarity index 100% rename from test/pytorch/sentiment-analyzer/cortex.yaml rename to test/apis/pytorch/sentiment-analyzer/cortex.yaml diff --git a/test/pytorch/sentiment-analyzer/predictor.py b/test/apis/pytorch/sentiment-analyzer/predictor.py similarity index 100% rename from test/pytorch/sentiment-analyzer/predictor.py rename to test/apis/pytorch/sentiment-analyzer/predictor.py diff --git a/test/pytorch/sentiment-analyzer/requirements.txt b/test/apis/pytorch/sentiment-analyzer/requirements.txt similarity index 100% rename from test/pytorch/sentiment-analyzer/requirements.txt rename to test/apis/pytorch/sentiment-analyzer/requirements.txt diff --git a/test/pytorch/sentiment-analyzer/sample.json b/test/apis/pytorch/sentiment-analyzer/sample.json similarity index 100% rename from test/pytorch/sentiment-analyzer/sample.json rename to test/apis/pytorch/sentiment-analyzer/sample.json diff --git a/test/pytorch/text-generator/README.md b/test/apis/pytorch/text-generator/README.md similarity index 100% rename from test/pytorch/text-generator/README.md rename to test/apis/pytorch/text-generator/README.md diff --git a/test/pytorch/text-generator/cortex.yaml b/test/apis/pytorch/text-generator/cortex.yaml similarity index 100% rename from test/pytorch/text-generator/cortex.yaml rename to test/apis/pytorch/text-generator/cortex.yaml diff --git a/test/pytorch/text-generator/deploy.ipynb b/test/apis/pytorch/text-generator/deploy.ipynb similarity index 100% rename from test/pytorch/text-generator/deploy.ipynb rename to test/apis/pytorch/text-generator/deploy.ipynb diff --git a/test/pytorch/text-generator/predictor.py b/test/apis/pytorch/text-generator/predictor.py similarity index 100% rename from test/pytorch/text-generator/predictor.py rename to test/apis/pytorch/text-generator/predictor.py diff --git a/test/pytorch/text-generator/requirements.txt b/test/apis/pytorch/text-generator/requirements.txt similarity index 100% rename from test/pytorch/text-generator/requirements.txt rename to test/apis/pytorch/text-generator/requirements.txt diff --git a/test/pytorch/text-generator/sample.json b/test/apis/pytorch/text-generator/sample.json similarity index 100% rename from test/pytorch/text-generator/sample.json rename to test/apis/pytorch/text-generator/sample.json diff --git a/test/pytorch/text-summarizer/README.md b/test/apis/pytorch/text-summarizer/README.md similarity index 100% rename from test/pytorch/text-summarizer/README.md rename to test/apis/pytorch/text-summarizer/README.md diff --git a/test/pytorch/text-summarizer/cortex.yaml b/test/apis/pytorch/text-summarizer/cortex.yaml similarity index 100% rename from test/pytorch/text-summarizer/cortex.yaml rename to test/apis/pytorch/text-summarizer/cortex.yaml diff --git a/test/pytorch/text-summarizer/predictor.py b/test/apis/pytorch/text-summarizer/predictor.py similarity index 100% rename from test/pytorch/text-summarizer/predictor.py rename to test/apis/pytorch/text-summarizer/predictor.py diff --git a/test/pytorch/text-summarizer/requirements.txt b/test/apis/pytorch/text-summarizer/requirements.txt similarity index 100% rename from test/pytorch/text-summarizer/requirements.txt rename to test/apis/pytorch/text-summarizer/requirements.txt diff --git a/test/pytorch/text-summarizer/sample.json b/test/apis/pytorch/text-summarizer/sample.json similarity index 100% rename from test/pytorch/text-summarizer/sample.json rename to test/apis/pytorch/text-summarizer/sample.json diff --git a/test/sklearn/iris-classifier/cortex.yaml b/test/apis/sklearn/iris-classifier/cortex.yaml similarity index 100% rename from test/sklearn/iris-classifier/cortex.yaml rename to test/apis/sklearn/iris-classifier/cortex.yaml diff --git a/test/sklearn/iris-classifier/predictor.py b/test/apis/sklearn/iris-classifier/predictor.py similarity index 100% rename from test/sklearn/iris-classifier/predictor.py rename to test/apis/sklearn/iris-classifier/predictor.py diff --git a/test/sklearn/iris-classifier/requirements.txt b/test/apis/sklearn/iris-classifier/requirements.txt similarity index 100% rename from test/sklearn/iris-classifier/requirements.txt rename to test/apis/sklearn/iris-classifier/requirements.txt diff --git a/test/sklearn/iris-classifier/sample.json b/test/apis/sklearn/iris-classifier/sample.json similarity index 100% rename from test/sklearn/iris-classifier/sample.json rename to test/apis/sklearn/iris-classifier/sample.json diff --git a/test/sklearn/iris-classifier/trainer.py b/test/apis/sklearn/iris-classifier/trainer.py similarity index 100% rename from test/sklearn/iris-classifier/trainer.py rename to test/apis/sklearn/iris-classifier/trainer.py diff --git a/test/sklearn/mpg-estimator/cortex.yaml b/test/apis/sklearn/mpg-estimator/cortex.yaml similarity index 100% rename from test/sklearn/mpg-estimator/cortex.yaml rename to test/apis/sklearn/mpg-estimator/cortex.yaml diff --git a/test/sklearn/mpg-estimator/predictor.py b/test/apis/sklearn/mpg-estimator/predictor.py similarity index 100% rename from test/sklearn/mpg-estimator/predictor.py rename to test/apis/sklearn/mpg-estimator/predictor.py diff --git a/test/sklearn/mpg-estimator/requirements.txt b/test/apis/sklearn/mpg-estimator/requirements.txt similarity index 100% rename from test/sklearn/mpg-estimator/requirements.txt rename to test/apis/sklearn/mpg-estimator/requirements.txt diff --git a/test/sklearn/mpg-estimator/sample.json b/test/apis/sklearn/mpg-estimator/sample.json similarity index 100% rename from test/sklearn/mpg-estimator/sample.json rename to test/apis/sklearn/mpg-estimator/sample.json diff --git a/test/sklearn/mpg-estimator/trainer.py b/test/apis/sklearn/mpg-estimator/trainer.py similarity index 100% rename from test/sklearn/mpg-estimator/trainer.py rename to test/apis/sklearn/mpg-estimator/trainer.py diff --git a/test/spacy/entity-recognizer/cortex.yaml b/test/apis/spacy/entity-recognizer/cortex.yaml similarity index 100% rename from test/spacy/entity-recognizer/cortex.yaml rename to test/apis/spacy/entity-recognizer/cortex.yaml diff --git a/test/spacy/entity-recognizer/predictor.py b/test/apis/spacy/entity-recognizer/predictor.py similarity index 100% rename from test/spacy/entity-recognizer/predictor.py rename to test/apis/spacy/entity-recognizer/predictor.py diff --git a/test/spacy/entity-recognizer/requirements.txt b/test/apis/spacy/entity-recognizer/requirements.txt similarity index 100% rename from test/spacy/entity-recognizer/requirements.txt rename to test/apis/spacy/entity-recognizer/requirements.txt diff --git a/test/spacy/entity-recognizer/sample.json b/test/apis/spacy/entity-recognizer/sample.json similarity index 100% rename from test/spacy/entity-recognizer/sample.json rename to test/apis/spacy/entity-recognizer/sample.json diff --git a/test/tensorflow/image-classifier-inception/cortex.yaml b/test/apis/tensorflow/image-classifier-inception/cortex.yaml similarity index 100% rename from test/tensorflow/image-classifier-inception/cortex.yaml rename to test/apis/tensorflow/image-classifier-inception/cortex.yaml diff --git a/test/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml similarity index 100% rename from test/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml rename to test/apis/tensorflow/image-classifier-inception/cortex_server_side_batching.yaml diff --git a/test/tensorflow/image-classifier-inception/inception.ipynb b/test/apis/tensorflow/image-classifier-inception/inception.ipynb similarity index 100% rename from test/tensorflow/image-classifier-inception/inception.ipynb rename to test/apis/tensorflow/image-classifier-inception/inception.ipynb diff --git a/test/tensorflow/image-classifier-inception/predictor.py b/test/apis/tensorflow/image-classifier-inception/predictor.py similarity index 100% rename from test/tensorflow/image-classifier-inception/predictor.py rename to test/apis/tensorflow/image-classifier-inception/predictor.py diff --git a/test/tensorflow/image-classifier-inception/requirements.txt b/test/apis/tensorflow/image-classifier-inception/requirements.txt similarity index 100% rename from test/tensorflow/image-classifier-inception/requirements.txt rename to test/apis/tensorflow/image-classifier-inception/requirements.txt diff --git a/test/tensorflow/image-classifier-inception/sample.json b/test/apis/tensorflow/image-classifier-inception/sample.json similarity index 100% rename from test/tensorflow/image-classifier-inception/sample.json rename to test/apis/tensorflow/image-classifier-inception/sample.json diff --git a/test/tensorflow/image-classifier-resnet50/README.md b/test/apis/tensorflow/image-classifier-resnet50/README.md similarity index 100% rename from test/tensorflow/image-classifier-resnet50/README.md rename to test/apis/tensorflow/image-classifier-resnet50/README.md diff --git a/test/tensorflow/image-classifier-resnet50/cortex.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_gpu.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_gpu.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_gpu.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_gpu_server_side_batching.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_inf.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_inf.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_inf.yaml diff --git a/test/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml b/test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml similarity index 100% rename from test/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml rename to test/apis/tensorflow/image-classifier-resnet50/cortex_inf_server_side_batching.yaml diff --git a/test/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb b/test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb similarity index 100% rename from test/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb rename to test/apis/tensorflow/image-classifier-resnet50/generate_gpu_resnet50_model.ipynb diff --git a/test/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb b/test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb similarity index 100% rename from test/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb rename to test/apis/tensorflow/image-classifier-resnet50/generate_resnet50_models.ipynb diff --git a/test/tensorflow/image-classifier-resnet50/predictor.py b/test/apis/tensorflow/image-classifier-resnet50/predictor.py similarity index 100% rename from test/tensorflow/image-classifier-resnet50/predictor.py rename to test/apis/tensorflow/image-classifier-resnet50/predictor.py diff --git a/test/tensorflow/image-classifier-resnet50/requirements.txt b/test/apis/tensorflow/image-classifier-resnet50/requirements.txt similarity index 100% rename from test/tensorflow/image-classifier-resnet50/requirements.txt rename to test/apis/tensorflow/image-classifier-resnet50/requirements.txt diff --git a/test/tensorflow/image-classifier-resnet50/sample.bin b/test/apis/tensorflow/image-classifier-resnet50/sample.bin similarity index 100% rename from test/tensorflow/image-classifier-resnet50/sample.bin rename to test/apis/tensorflow/image-classifier-resnet50/sample.bin diff --git a/test/tensorflow/image-classifier-resnet50/sample.json b/test/apis/tensorflow/image-classifier-resnet50/sample.json similarity index 100% rename from test/tensorflow/image-classifier-resnet50/sample.json rename to test/apis/tensorflow/image-classifier-resnet50/sample.json diff --git a/test/tensorflow/iris-classifier/cortex.yaml b/test/apis/tensorflow/iris-classifier/cortex.yaml similarity index 100% rename from test/tensorflow/iris-classifier/cortex.yaml rename to test/apis/tensorflow/iris-classifier/cortex.yaml diff --git a/test/tensorflow/iris-classifier/predictor.py b/test/apis/tensorflow/iris-classifier/predictor.py similarity index 100% rename from test/tensorflow/iris-classifier/predictor.py rename to test/apis/tensorflow/iris-classifier/predictor.py diff --git a/test/tensorflow/iris-classifier/sample.json b/test/apis/tensorflow/iris-classifier/sample.json similarity index 100% rename from test/tensorflow/iris-classifier/sample.json rename to test/apis/tensorflow/iris-classifier/sample.json diff --git a/test/tensorflow/iris-classifier/tensorflow.ipynb b/test/apis/tensorflow/iris-classifier/tensorflow.ipynb similarity index 100% rename from test/tensorflow/iris-classifier/tensorflow.ipynb rename to test/apis/tensorflow/iris-classifier/tensorflow.ipynb diff --git a/test/tensorflow/license-plate-reader/README.md b/test/apis/tensorflow/license-plate-reader/README.md similarity index 100% rename from test/tensorflow/license-plate-reader/README.md rename to test/apis/tensorflow/license-plate-reader/README.md diff --git a/test/tensorflow/license-plate-reader/config.json b/test/apis/tensorflow/license-plate-reader/config.json similarity index 100% rename from test/tensorflow/license-plate-reader/config.json rename to test/apis/tensorflow/license-plate-reader/config.json diff --git a/test/tensorflow/license-plate-reader/cortex_full.yaml b/test/apis/tensorflow/license-plate-reader/cortex_full.yaml similarity index 100% rename from test/tensorflow/license-plate-reader/cortex_full.yaml rename to test/apis/tensorflow/license-plate-reader/cortex_full.yaml diff --git a/test/tensorflow/license-plate-reader/cortex_lite.yaml b/test/apis/tensorflow/license-plate-reader/cortex_lite.yaml similarity index 100% rename from test/tensorflow/license-plate-reader/cortex_lite.yaml rename to test/apis/tensorflow/license-plate-reader/cortex_lite.yaml diff --git a/test/tensorflow/license-plate-reader/predictor_crnn.py b/test/apis/tensorflow/license-plate-reader/predictor_crnn.py similarity index 100% rename from test/tensorflow/license-plate-reader/predictor_crnn.py rename to test/apis/tensorflow/license-plate-reader/predictor_crnn.py diff --git a/test/tensorflow/license-plate-reader/predictor_lite.py b/test/apis/tensorflow/license-plate-reader/predictor_lite.py similarity index 100% rename from test/tensorflow/license-plate-reader/predictor_lite.py rename to test/apis/tensorflow/license-plate-reader/predictor_lite.py diff --git a/test/tensorflow/license-plate-reader/predictor_yolo.py b/test/apis/tensorflow/license-plate-reader/predictor_yolo.py similarity index 100% rename from test/tensorflow/license-plate-reader/predictor_yolo.py rename to test/apis/tensorflow/license-plate-reader/predictor_yolo.py diff --git a/test/tensorflow/license-plate-reader/requirements.txt b/test/apis/tensorflow/license-plate-reader/requirements.txt similarity index 100% rename from test/tensorflow/license-plate-reader/requirements.txt rename to test/apis/tensorflow/license-plate-reader/requirements.txt diff --git a/test/tensorflow/license-plate-reader/sample_inference.py b/test/apis/tensorflow/license-plate-reader/sample_inference.py similarity index 100% rename from test/tensorflow/license-plate-reader/sample_inference.py rename to test/apis/tensorflow/license-plate-reader/sample_inference.py diff --git a/test/tensorflow/license-plate-reader/utils/__init__.py b/test/apis/tensorflow/license-plate-reader/utils/__init__.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/__init__.py rename to test/apis/tensorflow/license-plate-reader/utils/__init__.py diff --git a/test/tensorflow/license-plate-reader/utils/bbox.py b/test/apis/tensorflow/license-plate-reader/utils/bbox.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/bbox.py rename to test/apis/tensorflow/license-plate-reader/utils/bbox.py diff --git a/test/tensorflow/license-plate-reader/utils/colors.py b/test/apis/tensorflow/license-plate-reader/utils/colors.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/colors.py rename to test/apis/tensorflow/license-plate-reader/utils/colors.py diff --git a/test/tensorflow/license-plate-reader/utils/preprocess.py b/test/apis/tensorflow/license-plate-reader/utils/preprocess.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/preprocess.py rename to test/apis/tensorflow/license-plate-reader/utils/preprocess.py diff --git a/test/tensorflow/license-plate-reader/utils/utils.py b/test/apis/tensorflow/license-plate-reader/utils/utils.py similarity index 100% rename from test/tensorflow/license-plate-reader/utils/utils.py rename to test/apis/tensorflow/license-plate-reader/utils/utils.py diff --git a/test/tensorflow/multi-model-classifier/README.md b/test/apis/tensorflow/multi-model-classifier/README.md similarity index 100% rename from test/tensorflow/multi-model-classifier/README.md rename to test/apis/tensorflow/multi-model-classifier/README.md diff --git a/test/tensorflow/multi-model-classifier/cortex.yaml b/test/apis/tensorflow/multi-model-classifier/cortex.yaml similarity index 100% rename from test/tensorflow/multi-model-classifier/cortex.yaml rename to test/apis/tensorflow/multi-model-classifier/cortex.yaml diff --git a/test/tensorflow/multi-model-classifier/predictor.py b/test/apis/tensorflow/multi-model-classifier/predictor.py similarity index 100% rename from test/tensorflow/multi-model-classifier/predictor.py rename to test/apis/tensorflow/multi-model-classifier/predictor.py diff --git a/test/tensorflow/multi-model-classifier/requirements.txt b/test/apis/tensorflow/multi-model-classifier/requirements.txt similarity index 100% rename from test/tensorflow/multi-model-classifier/requirements.txt rename to test/apis/tensorflow/multi-model-classifier/requirements.txt diff --git a/test/tensorflow/multi-model-classifier/sample-image.json b/test/apis/tensorflow/multi-model-classifier/sample-image.json similarity index 100% rename from test/tensorflow/multi-model-classifier/sample-image.json rename to test/apis/tensorflow/multi-model-classifier/sample-image.json diff --git a/test/tensorflow/multi-model-classifier/sample-iris.json b/test/apis/tensorflow/multi-model-classifier/sample-iris.json similarity index 100% rename from test/tensorflow/multi-model-classifier/sample-iris.json rename to test/apis/tensorflow/multi-model-classifier/sample-iris.json diff --git a/test/tensorflow/sentiment-analyzer/bert.ipynb b/test/apis/tensorflow/sentiment-analyzer/bert.ipynb similarity index 100% rename from test/tensorflow/sentiment-analyzer/bert.ipynb rename to test/apis/tensorflow/sentiment-analyzer/bert.ipynb diff --git a/test/tensorflow/sentiment-analyzer/cortex.yaml b/test/apis/tensorflow/sentiment-analyzer/cortex.yaml similarity index 100% rename from test/tensorflow/sentiment-analyzer/cortex.yaml rename to test/apis/tensorflow/sentiment-analyzer/cortex.yaml diff --git a/test/tensorflow/sentiment-analyzer/predictor.py b/test/apis/tensorflow/sentiment-analyzer/predictor.py similarity index 100% rename from test/tensorflow/sentiment-analyzer/predictor.py rename to test/apis/tensorflow/sentiment-analyzer/predictor.py diff --git a/test/tensorflow/sentiment-analyzer/requirements.txt b/test/apis/tensorflow/sentiment-analyzer/requirements.txt similarity index 100% rename from test/tensorflow/sentiment-analyzer/requirements.txt rename to test/apis/tensorflow/sentiment-analyzer/requirements.txt diff --git a/test/tensorflow/sentiment-analyzer/sample.json b/test/apis/tensorflow/sentiment-analyzer/sample.json similarity index 100% rename from test/tensorflow/sentiment-analyzer/sample.json rename to test/apis/tensorflow/sentiment-analyzer/sample.json diff --git a/test/tensorflow/text-generator/cortex.yaml b/test/apis/tensorflow/text-generator/cortex.yaml similarity index 100% rename from test/tensorflow/text-generator/cortex.yaml rename to test/apis/tensorflow/text-generator/cortex.yaml diff --git a/test/tensorflow/text-generator/encoder.py b/test/apis/tensorflow/text-generator/encoder.py similarity index 100% rename from test/tensorflow/text-generator/encoder.py rename to test/apis/tensorflow/text-generator/encoder.py diff --git a/test/tensorflow/text-generator/gpt-2.ipynb b/test/apis/tensorflow/text-generator/gpt-2.ipynb similarity index 100% rename from test/tensorflow/text-generator/gpt-2.ipynb rename to test/apis/tensorflow/text-generator/gpt-2.ipynb diff --git a/test/tensorflow/text-generator/predictor.py b/test/apis/tensorflow/text-generator/predictor.py similarity index 100% rename from test/tensorflow/text-generator/predictor.py rename to test/apis/tensorflow/text-generator/predictor.py diff --git a/test/tensorflow/text-generator/requirements.txt b/test/apis/tensorflow/text-generator/requirements.txt similarity index 100% rename from test/tensorflow/text-generator/requirements.txt rename to test/apis/tensorflow/text-generator/requirements.txt diff --git a/test/tensorflow/text-generator/sample.json b/test/apis/tensorflow/text-generator/sample.json similarity index 100% rename from test/tensorflow/text-generator/sample.json rename to test/apis/tensorflow/text-generator/sample.json diff --git a/test/traffic-splitter/README.md b/test/apis/traffic-splitter/README.md similarity index 100% rename from test/traffic-splitter/README.md rename to test/apis/traffic-splitter/README.md diff --git a/test/traffic-splitter/cortex.yaml b/test/apis/traffic-splitter/cortex.yaml similarity index 100% rename from test/traffic-splitter/cortex.yaml rename to test/apis/traffic-splitter/cortex.yaml diff --git a/test/traffic-splitter/model.py b/test/apis/traffic-splitter/model.py similarity index 100% rename from test/traffic-splitter/model.py rename to test/apis/traffic-splitter/model.py diff --git a/test/traffic-splitter/onnx_predictor.py b/test/apis/traffic-splitter/onnx_predictor.py similarity index 100% rename from test/traffic-splitter/onnx_predictor.py rename to test/apis/traffic-splitter/onnx_predictor.py diff --git a/test/traffic-splitter/pytorch_predictor.py b/test/apis/traffic-splitter/pytorch_predictor.py similarity index 100% rename from test/traffic-splitter/pytorch_predictor.py rename to test/apis/traffic-splitter/pytorch_predictor.py diff --git a/test/traffic-splitter/sample.json b/test/apis/traffic-splitter/sample.json similarity index 100% rename from test/traffic-splitter/sample.json rename to test/apis/traffic-splitter/sample.json From f49304713e50a307e3d2d171393f69676648e3df Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 10 Dec 2020 14:33:27 +0100 Subject: [PATCH 02/31] Add E2E test package --- test/e2e/e2e/__init__.py | 17 ++++++++ test/e2e/e2e/cluster.py | 42 ++++++++++++++++++ test/e2e/e2e/exceptions.py | 10 +++++ test/e2e/e2e/expectations.py | 83 ++++++++++++++++++++++++++++++++++++ test/e2e/e2e/utils.py | 45 +++++++++++++++++++ test/e2e/setup.py | 37 ++++++++++++++++ 6 files changed, 234 insertions(+) create mode 100644 test/e2e/e2e/__init__.py create mode 100644 test/e2e/e2e/cluster.py create mode 100644 test/e2e/e2e/exceptions.py create mode 100644 test/e2e/e2e/expectations.py create mode 100644 test/e2e/e2e/utils.py create mode 100644 test/e2e/setup.py diff --git a/test/e2e/e2e/__init__.py b/test/e2e/e2e/__init__.py new file mode 100644 index 0000000000..181f818dd1 --- /dev/null +++ b/test/e2e/e2e/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .cluster import create_cluster, delete_cluster + +__all__ = ["create_cluster", "delete_cluster"] diff --git a/test/e2e/e2e/cluster.py b/test/e2e/e2e/cluster.py new file mode 100644 index 0000000000..5c89d9c142 --- /dev/null +++ b/test/e2e/e2e/cluster.py @@ -0,0 +1,42 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys + +from e2e.exceptions import ClusterCreationException, ClusterDeletionException + + +def create_cluster(cluster_config: str): + """Create a cortex cluster from a cluster config""" + p = subprocess.run( + ["cortex", "cluster", "up", "--config", cluster_config], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + if p.returncode != 0: + raise ClusterCreationException(f"failed to create cluster with config: {cluster_config}") + + +def delete_cluster(cluster_config: str): + """Delete a cortex cluster from a cluster config""" + p = subprocess.run( + ["cortex", "cluster", "down", "--config", cluster_config], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + if p.returncode != 0: + raise ClusterDeletionException(f"failed to delete cluster with config: {cluster_config}") diff --git a/test/e2e/e2e/exceptions.py b/test/e2e/e2e/exceptions.py new file mode 100644 index 0000000000..cc5710290b --- /dev/null +++ b/test/e2e/e2e/exceptions.py @@ -0,0 +1,10 @@ +class ClusterCreationException(Exception): + pass + + +class ClusterDeletionException(Exception): + pass + + +class ExpectationsValidationException(Exception): + pass diff --git a/test/e2e/e2e/expectations.py b/test/e2e/e2e/expectations.py new file mode 100644 index 0000000000..500723e047 --- /dev/null +++ b/test/e2e/e2e/expectations.py @@ -0,0 +1,83 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import types +from typing import Dict, Any + +import jsonschema +import requests +import yaml +from jsonschema import Draft7Validator + +from e2e.exceptions import ExpectationsValidationException + +CONTENT_TO_ATTR = {"text": "text", "json": "json", "binary": "content"} + + +def assert_response_expectations(response: requests.Response, expectations: Dict[str, Any]): + content_type = expectations["content_type"] + + expected = expectations.get("expected") + if expected: + output = _get_response_content(response, content_type) + assert output == expected, f"expected response: got {output}, expected {expected}" + + expected_json_schema = expectations.get("json_schema") + if expected_json_schema: + output = _get_response_content(response, content_type) + jsonschema.validate(output, schema=expected_json_schema) + + +def parse_expectations(expectations_file: str) -> Dict[str, Any]: + with open(expectations_file) as f: + expectations = yaml.safe_load(f) + + validate_expectations(expectations) + + return expectations + + +def validate_expectations(expectations): + if "response" in expectations: + validate_response_expectations(expectations["response"]) + + +def validate_response_expectations(expectations: Dict[str, Any]): + if not expectations["content_type"] in CONTENT_TO_ATTR.keys(): + raise ExpectationsValidationException( + f"response.content_type should be one of {CONTENT_TO_ATTR.keys()}" + ) + + if "expected" in expectations and "json_schema" in expectations: + raise ExpectationsValidationException("expected and json_schema are mutually exclusive") + + if "json_schema" in expectations: + if expectations["content_type"] != "json": + raise ExpectationsValidationException( + "json_schema is only valid when content_type is set to json" + ) + + try: + Draft7Validator.check_schema(schema=expectations["json_schema"]) + except Exception as e: + raise ExpectationsValidationException("json_schema is invalid") from e + + +def _get_response_content(response: requests.Response, content_type: str) -> str: + attr = CONTENT_TO_ATTR.get(content_type, "content") + content = getattr(response, attr) + if isinstance(content, types.MethodType): + return content() + + return content diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py new file mode 100644 index 0000000000..fcefdc8f43 --- /dev/null +++ b/test/e2e/e2e/utils.py @@ -0,0 +1,45 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from typing import List, Optional, Dict, Union + +import cortex as cx +import requests + + +def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = None) -> bool: + count = 0 + while True: + if timeout is not None and count > timeout: + return False + + ready = all( + [client.get_api(name)["status"]["status_code"] == "status_live" for name in api_names] + ) + if ready: + return True + + time.sleep(1) + count += 1 + + +def request_prediction( + client: cx.Client, api_name: str, payload: Union[List, Dict] +) -> requests.Response: + + api_info = client.get_api(api_name) + response = requests.post(api_info["endpoint"], json=payload) + + return response diff --git a/test/e2e/setup.py b/test/e2e/setup.py new file mode 100644 index 0000000000..699d4a4ddf --- /dev/null +++ b/test/e2e/setup.py @@ -0,0 +1,37 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from setuptools import setup, find_packages + +root = Path(__file__).parent.absolute() +cortex_client_dir = root.parent.parent / "pkg" / "workloads" / "cortex" / "client" + +if not cortex_client_dir.exists(): + raise ModuleNotFoundError(f"cortex client not found in {cortex_client_dir}") + +setup( + name="e2e", + version="master", + packages=find_packages(), + url="", + license="Apache 2.0 Licence", + python_requires=">=3.6", + install_requires=["requests", "cortex"], + dependency_links=[f"file://{cortex_client_dir}#egg=cortex"], + author="Cortex Labs", + author_email="hello@cortexlabs.com", + description="Cortex E2E tests package", +) From e699b5bca78a65a6716e25920f9d1aff34df2722 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 10 Dec 2020 14:33:50 +0100 Subject: [PATCH 03/31] Add e2e test for local realtime apis --- test/e2e/tests/local/test_realtime.py | 73 +++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 test/e2e/tests/local/test_realtime.py diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py new file mode 100644 index 0000000000..b775aa1624 --- /dev/null +++ b/test/e2e/tests/local/test_realtime.py @@ -0,0 +1,73 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from http import HTTPStatus +from pathlib import Path +from typing import List + +import cortex as cx +import pytest +import yaml + +from e2e.expectations import assert_response_expectations, parse_expectations +from e2e.utils import apis_ready, request_prediction + +TEST_APIS_DIR = Path(__file__).parent.parent.parent.parent / "apis" +TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] +DEPLOY_TIMEOUT = 30 # seconds + + +@pytest.fixture +def client(): + return cx.client("local") + + +def delete_apis(client: cx.Client, api_names: List[str]): + for name in api_names: + client.delete_api(name) + + +@pytest.mark.parametrize("api", TEST_APIS) +def test_realtime_apis(client: cx.Client, api: str): + api_dir = TEST_APIS_DIR / api + with open(str(api_dir / "cortex.yaml")) as f: + api_specs = yaml.safe_load(f) + + expectations = None + expectations_file = api_dir / "expectations.yaml" + if expectations_file.exists(): + expectations = parse_expectations(str(expectations_file)) + + api_names = [api_spec["name"] for api_spec in api_specs] + for api_spec in api_specs: + client.create_api(api_spec=api_spec, project_dir=api_dir) + + try: + assert apis_ready(client=client, api_names=api_names), f"apis {api_names} not ready" + + with open(str(api_dir / "sample.json")) as f: + payload = json.load(f) + + api_name = api_names[0] + response = request_prediction(client, api_name, payload) + + assert ( + response.status_code == HTTPStatus.OK + ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" + + if expectations and "response" in expectations: + assert_response_expectations(response, expectations["response"]) + finally: + delete_apis(client, api_names) From 5b35b543ceb0037af51384613de56db0b8305d4a Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 11 Dec 2020 11:25:25 +0100 Subject: [PATCH 04/31] Add -y flag to cluster management function --- test/e2e/e2e/cluster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e/cluster.py b/test/e2e/e2e/cluster.py index 5c89d9c142..9ba544099a 100644 --- a/test/e2e/e2e/cluster.py +++ b/test/e2e/e2e/cluster.py @@ -21,7 +21,7 @@ def create_cluster(cluster_config: str): """Create a cortex cluster from a cluster config""" p = subprocess.run( - ["cortex", "cluster", "up", "--config", cluster_config], + ["cortex", "cluster", "up", "-y", "--config", cluster_config], stdout=sys.stdout, stderr=sys.stderr, ) @@ -33,7 +33,7 @@ def create_cluster(cluster_config: str): def delete_cluster(cluster_config: str): """Delete a cortex cluster from a cluster config""" p = subprocess.run( - ["cortex", "cluster", "down", "--config", cluster_config], + ["cortex", "cluster", "down", "-y", "--config", cluster_config], stdout=sys.stdout, stderr=sys.stderr, ) From 4782de531181808b20b057f0a2e3a2fa6a702b4b Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 11 Dec 2020 11:26:24 +0100 Subject: [PATCH 05/31] Refactor test file in order to share test with different envs --- test/e2e/e2e/tests.py | 63 +++++++++++++++++++++++++++ test/e2e/tests/local/test_realtime.py | 46 +------------------ 2 files changed, 65 insertions(+), 44 deletions(-) create mode 100644 test/e2e/e2e/tests.py diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py new file mode 100644 index 0000000000..d005ca2102 --- /dev/null +++ b/test/e2e/e2e/tests.py @@ -0,0 +1,63 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from http import HTTPStatus +from pathlib import Path +from typing import List + +import cortex as cx +import yaml + +from e2e.expectations import parse_expectations, assert_response_expectations +from e2e.utils import apis_ready, request_prediction + +TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" + + +def delete_apis(client: cx.Client, api_names: List[str]): + for name in api_names: + client.delete_api(name) + + +def test_realtime_apis(client: cx.Client, api: str, timeout: int = None): + api_dir = TEST_APIS_DIR / api + with open(str(api_dir / "cortex.yaml")) as f: + api_specs = yaml.safe_load(f) + + expectations = None + expectations_file = api_dir / "expectations.yaml" + if expectations_file.exists(): + expectations = parse_expectations(str(expectations_file)) + + api_names = [api_spec["name"] for api_spec in api_specs] + for api_spec in api_specs: + client.create_api(api_spec=api_spec, project_dir=api_dir) + + try: + assert apis_ready(client=client, api_names=api_names, timeout=timeout), f"apis {api_names} not ready" + + with open(str(api_dir / "sample.json")) as f: + payload = json.load(f) + + api_name = api_names[0] + response = request_prediction(client, api_name, payload) + + assert ( + response.status_code == HTTPStatus.OK + ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" + + if expectations and "response" in expectations: + assert_response_expectations(response, expectations["response"]) + finally: + delete_apis(client, api_names) diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py index b775aa1624..edc58e6422 100644 --- a/test/e2e/tests/local/test_realtime.py +++ b/test/e2e/tests/local/test_realtime.py @@ -12,19 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -from http import HTTPStatus -from pathlib import Path -from typing import List - import cortex as cx import pytest -import yaml -from e2e.expectations import assert_response_expectations, parse_expectations -from e2e.utils import apis_ready, request_prediction +import e2e.tests -TEST_APIS_DIR = Path(__file__).parent.parent.parent.parent / "apis" TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] DEPLOY_TIMEOUT = 30 # seconds @@ -34,40 +26,6 @@ def client(): return cx.client("local") -def delete_apis(client: cx.Client, api_names: List[str]): - for name in api_names: - client.delete_api(name) - - @pytest.mark.parametrize("api", TEST_APIS) def test_realtime_apis(client: cx.Client, api: str): - api_dir = TEST_APIS_DIR / api - with open(str(api_dir / "cortex.yaml")) as f: - api_specs = yaml.safe_load(f) - - expectations = None - expectations_file = api_dir / "expectations.yaml" - if expectations_file.exists(): - expectations = parse_expectations(str(expectations_file)) - - api_names = [api_spec["name"] for api_spec in api_specs] - for api_spec in api_specs: - client.create_api(api_spec=api_spec, project_dir=api_dir) - - try: - assert apis_ready(client=client, api_names=api_names), f"apis {api_names} not ready" - - with open(str(api_dir / "sample.json")) as f: - payload = json.load(f) - - api_name = api_names[0] - response = request_prediction(client, api_name, payload) - - assert ( - response.status_code == HTTPStatus.OK - ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" - - if expectations and "response" in expectations: - assert_response_expectations(response, expectations["response"]) - finally: - delete_apis(client, api_names) + e2e.tests.test_realtime_apis(client=client, api=api, timeout=DEPLOY_TIMEOUT) From 6e2c1e86eecd4f75f4dbc9b7cde58927b6381276 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 11 Dec 2020 11:27:13 +0100 Subject: [PATCH 06/31] Add pytest.ini configuration --- test/e2e/pytest.ini | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 test/e2e/pytest.ini diff --git a/test/e2e/pytest.ini b/test/e2e/pytest.ini new file mode 100644 index 0000000000..620bae68f9 --- /dev/null +++ b/test/e2e/pytest.ini @@ -0,0 +1,4 @@ +# pytest.ini +[pytest] +minversion = 6.0 +addopts = -s -v From b5b4f6b43c43ff2d168903c17712cbc3db31a786 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 11 Dec 2020 15:40:24 +0100 Subject: [PATCH 07/31] Rename test function --- test/e2e/e2e/tests.py | 6 ++++-- test/e2e/tests/local/test_realtime.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index d005ca2102..6b8d5f3ba2 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -30,7 +30,7 @@ def delete_apis(client: cx.Client, api_names: List[str]): client.delete_api(name) -def test_realtime_apis(client: cx.Client, api: str, timeout: int = None): +def test_realtime_api(client: cx.Client, api: str, timeout: int = None): api_dir = TEST_APIS_DIR / api with open(str(api_dir / "cortex.yaml")) as f: api_specs = yaml.safe_load(f) @@ -45,7 +45,9 @@ def test_realtime_apis(client: cx.Client, api: str, timeout: int = None): client.create_api(api_spec=api_spec, project_dir=api_dir) try: - assert apis_ready(client=client, api_names=api_names, timeout=timeout), f"apis {api_names} not ready" + assert apis_ready( + client=client, api_names=api_names, timeout=timeout + ), f"apis {api_names} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py index edc58e6422..29a2db7071 100644 --- a/test/e2e/tests/local/test_realtime.py +++ b/test/e2e/tests/local/test_realtime.py @@ -28,4 +28,4 @@ def client(): @pytest.mark.parametrize("api", TEST_APIS) def test_realtime_apis(client: cx.Client, api: str): - e2e.tests.test_realtime_apis(client=client, api=api, timeout=DEPLOY_TIMEOUT) + e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) From 0c47cd117f6404e90d83616848f0779f06113f8a Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 11 Dec 2020 15:41:50 +0100 Subject: [PATCH 08/31] Add aws e2e tests for realtime api's --- test/e2e/e2e/cluster.py | 19 ++++++++++++- test/e2e/e2e/utils.py | 11 ++++++++ test/e2e/tests/aws/conftest.py | 32 ++++++++++++++++++++++ test/e2e/tests/aws/test_realtime.py | 42 +++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 test/e2e/tests/aws/conftest.py create mode 100644 test/e2e/tests/aws/test_realtime.py diff --git a/test/e2e/e2e/cluster.py b/test/e2e/e2e/cluster.py index 9ba544099a..f85217b7d0 100644 --- a/test/e2e/e2e/cluster.py +++ b/test/e2e/e2e/cluster.py @@ -15,13 +15,30 @@ import subprocess import sys +import yaml + from e2e.exceptions import ClusterCreationException, ClusterDeletionException def create_cluster(cluster_config: str): """Create a cortex cluster from a cluster config""" + with open(cluster_config) as f: + config = yaml.safe_load(f) + + cluster_name = config["cluster_name"] + provider = config["provider"] + p = subprocess.run( - ["cortex", "cluster", "up", "-y", "--config", cluster_config], + [ + "cortex", + "cluster", + "up", + "-y", + "--config", + cluster_config, + "--configure-env", + f"{cluster_name}-{provider}", + ], stdout=sys.stdout, stderr=sys.stderr, ) diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index fcefdc8f43..f9aad8e6aa 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -17,6 +17,7 @@ import cortex as cx import requests +import yaml def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = None) -> bool: @@ -43,3 +44,13 @@ def request_prediction( response = requests.post(api_info["endpoint"], json=payload) return response + + +def client_from_config(config_path: str) -> cx.Client: + with open(config_path) as f: + config = yaml.safe_load(f) + + cluster_name = config["cluster_name"] + provider = config["provider"] + + return cx.client(f"{cluster_name}-{provider}") diff --git a/test/e2e/tests/aws/conftest.py b/test/e2e/tests/aws/conftest.py new file mode 100644 index 0000000000..3b49135357 --- /dev/null +++ b/test/e2e/tests/aws/conftest.py @@ -0,0 +1,32 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import e2e + + +def pytest_addoption(parser): + parser.addoption("--aws-env", action="store", default=None) + parser.addoption("--aws-config", action="store", default=None) + + +def pytest_configure(config): + aws_config = config.getoption("--aws-config") + if aws_config: + e2e.create_cluster(aws_config) + + +def pytest_unconfigure(config): + aws_config = config.getoption("--aws-config") + if aws_config: + e2e.delete_cluster(aws_config) diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py new file mode 100644 index 0000000000..607f4ee357 --- /dev/null +++ b/test/e2e/tests/aws/test_realtime.py @@ -0,0 +1,42 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cortex as cx +import pytest + +import e2e.tests +from e2e.utils import client_from_config + +TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] +DEPLOY_TIMEOUT = 30 # seconds + + +@pytest.fixture +def client(request): + env_name = request.config.getoption("--aws-env") + if env_name: + return cx.client(env_name) + + config_path = request.config.getoption("--aws-config") + if config_path is None: + raise ValueError( + "missing arguments: --env-name or --aws-config " + ) + + return client_from_config(config_path) + + +@pytest.mark.parametrize("api", TEST_APIS) +def test_realtime_api(client: cx.Client, api: str): + e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) From 0dc928e094b7171dfc27b088eed3fd1c2237510a Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 11 Dec 2020 16:07:42 +0100 Subject: [PATCH 09/31] Add e2e tests for GCP realtime api's --- test/e2e/tests/gcp/conftest.py | 32 ++++++++++++++++++++++ test/e2e/tests/gcp/test_realtime.py | 42 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 test/e2e/tests/gcp/conftest.py create mode 100644 test/e2e/tests/gcp/test_realtime.py diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py new file mode 100644 index 0000000000..af6a0b79d6 --- /dev/null +++ b/test/e2e/tests/gcp/conftest.py @@ -0,0 +1,32 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import e2e + + +def pytest_addoption(parser): + parser.addoption("--gcp-env", action="store", default=None) + parser.addoption("--gcp-config", action="store", default=None) + + +def pytest_configure(config): + aws_config = config.getoption("--gcp-config") + if aws_config: + e2e.create_cluster(aws_config) + + +def pytest_unconfigure(config): + aws_config = config.getoption("--gcp-config") + if aws_config: + e2e.delete_cluster(aws_config) diff --git a/test/e2e/tests/gcp/test_realtime.py b/test/e2e/tests/gcp/test_realtime.py new file mode 100644 index 0000000000..3b7fb30ed3 --- /dev/null +++ b/test/e2e/tests/gcp/test_realtime.py @@ -0,0 +1,42 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cortex as cx +import pytest + +import e2e.tests +from e2e.utils import client_from_config + +TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] +DEPLOY_TIMEOUT = 30 # seconds + + +@pytest.fixture +def client(request): + env_name = request.config.getoption("--gcp-env") + if env_name: + return cx.client(env_name) + + config_path = request.config.getoption("--gcp-config") + if config_path is None: + raise ValueError( + "missing arguments: --env-name or --gcp-config " + ) + + return client_from_config(config_path) + + +@pytest.mark.parametrize("api", TEST_APIS) +def test_realtime_apis(client: cx.Client, api: str): + e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) From f4003102cf3c713d5387b457b4ab16ad10f849f4 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 14 Dec 2020 12:03:58 +0000 Subject: [PATCH 10/31] Fix pytest custom options --- test/e2e/tests/__init__.py | 13 +++++++++++ test/e2e/tests/aws/__init__.py | 13 +++++++++++ test/e2e/tests/aws/conftest.py | 5 ---- test/e2e/tests/conftest.py | 40 ++++++++++++++++++++++++++++++++ test/e2e/tests/gcp/__init__.py | 13 +++++++++++ test/e2e/tests/gcp/conftest.py | 5 ---- test/e2e/tests/local/__init__.py | 13 +++++++++++ 7 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 test/e2e/tests/__init__.py create mode 100644 test/e2e/tests/aws/__init__.py create mode 100644 test/e2e/tests/conftest.py create mode 100644 test/e2e/tests/gcp/__init__.py create mode 100644 test/e2e/tests/local/__init__.py diff --git a/test/e2e/tests/__init__.py b/test/e2e/tests/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/e2e/tests/aws/__init__.py b/test/e2e/tests/aws/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/aws/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/e2e/tests/aws/conftest.py b/test/e2e/tests/aws/conftest.py index 3b49135357..7fe06842ac 100644 --- a/test/e2e/tests/aws/conftest.py +++ b/test/e2e/tests/aws/conftest.py @@ -15,11 +15,6 @@ import e2e -def pytest_addoption(parser): - parser.addoption("--aws-env", action="store", default=None) - parser.addoption("--aws-config", action="store", default=None) - - def pytest_configure(config): aws_config = config.getoption("--aws-config") if aws_config: diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py new file mode 100644 index 0000000000..8182d74b12 --- /dev/null +++ b/test/e2e/tests/conftest.py @@ -0,0 +1,40 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def pytest_addoption(parser): + parser.addoption( + "--aws-env", + action="store", + default=None, + help="set cortex AWS environment, to test on an existing AWS cluster", + ) + parser.addoption( + "--aws-config", + action="store", + default=None, + help="set cortex AWS cluster config, to test on a new AWS cluster", + ) + parser.addoption( + "--gcp-env", + action="store", + default=None, + help="set cortex GCP environment, to test on an existing GCP cluster", + ) + parser.addoption( + "--gcp-config", + action="store", + default=None, + help="set cortex GCP cluster config, to test on a new GCP cluster", + ) diff --git a/test/e2e/tests/gcp/__init__.py b/test/e2e/tests/gcp/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/gcp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py index af6a0b79d6..94618b7482 100644 --- a/test/e2e/tests/gcp/conftest.py +++ b/test/e2e/tests/gcp/conftest.py @@ -15,11 +15,6 @@ import e2e -def pytest_addoption(parser): - parser.addoption("--gcp-env", action="store", default=None) - parser.addoption("--gcp-config", action="store", default=None) - - def pytest_configure(config): aws_config = config.getoption("--gcp-config") if aws_config: diff --git a/test/e2e/tests/local/__init__.py b/test/e2e/tests/local/__init__.py new file mode 100644 index 0000000000..ab054358f3 --- /dev/null +++ b/test/e2e/tests/local/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From b6106699f7599be28f3acc06f98b477dbfb30238 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 14 Dec 2020 22:28:14 +0000 Subject: [PATCH 11/31] Add more test cases and move client fixtures to conftest.py --- test/e2e/tests/aws/conftest.py | 19 +++++++++++++++++++ test/e2e/tests/aws/test_realtime.py | 19 ++----------------- test/e2e/tests/gcp/conftest.py | 19 +++++++++++++++++++ test/e2e/tests/gcp/test_realtime.py | 19 ++----------------- test/e2e/tests/local/test_realtime.py | 2 +- 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/test/e2e/tests/aws/conftest.py b/test/e2e/tests/aws/conftest.py index 7fe06842ac..0921243ffb 100644 --- a/test/e2e/tests/aws/conftest.py +++ b/test/e2e/tests/aws/conftest.py @@ -12,7 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import cortex as cx +import pytest + import e2e +from e2e.utils import client_from_config + + +@pytest.fixture +def client(request): + env_name = request.config.getoption("--aws-env") + if env_name: + return cx.client(env_name) + + config_path = request.config.getoption("--aws-config") + if config_path is None: + raise ValueError( + "missing arguments: --env-name or --aws-config " + ) + + return client_from_config(config_path) def pytest_configure(config): diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py index 607f4ee357..58c1e230da 100644 --- a/test/e2e/tests/aws/test_realtime.py +++ b/test/e2e/tests/aws/test_realtime.py @@ -16,27 +16,12 @@ import pytest import e2e.tests -from e2e.utils import client_from_config -TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] +TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] DEPLOY_TIMEOUT = 30 # seconds -@pytest.fixture -def client(request): - env_name = request.config.getoption("--aws-env") - if env_name: - return cx.client(env_name) - - config_path = request.config.getoption("--aws-config") - if config_path is None: - raise ValueError( - "missing arguments: --env-name or --aws-config " - ) - - return client_from_config(config_path) - - +@pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_realtime_api(client: cx.Client, api: str): e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py index 94618b7482..69b5d82eb7 100644 --- a/test/e2e/tests/gcp/conftest.py +++ b/test/e2e/tests/gcp/conftest.py @@ -12,7 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import cortex as cx +import pytest + import e2e +from e2e.utils import client_from_config + + +@pytest.fixture +def client(request): + env_name = request.config.getoption("--gcp-env") + if env_name: + return cx.client(env_name) + + config_path = request.config.getoption("--gcp-config") + if config_path is None: + raise ValueError( + "missing arguments: --env-name or --gcp-config " + ) + + return client_from_config(config_path) def pytest_configure(config): diff --git a/test/e2e/tests/gcp/test_realtime.py b/test/e2e/tests/gcp/test_realtime.py index 3b7fb30ed3..3769ae84be 100644 --- a/test/e2e/tests/gcp/test_realtime.py +++ b/test/e2e/tests/gcp/test_realtime.py @@ -16,27 +16,12 @@ import pytest import e2e.tests -from e2e.utils import client_from_config -TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] +TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] DEPLOY_TIMEOUT = 30 # seconds -@pytest.fixture -def client(request): - env_name = request.config.getoption("--gcp-env") - if env_name: - return cx.client(env_name) - - config_path = request.config.getoption("--gcp-config") - if config_path is None: - raise ValueError( - "missing arguments: --env-name or --gcp-config " - ) - - return client_from_config(config_path) - - +@pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_realtime_apis(client: cx.Client, api: str): e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py index 29a2db7071..1cd565691b 100644 --- a/test/e2e/tests/local/test_realtime.py +++ b/test/e2e/tests/local/test_realtime.py @@ -17,7 +17,7 @@ import e2e.tests -TEST_APIS = ["pytorch/iris-classifier", "pytorch/image-classifier-resnet50"] +TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] DEPLOY_TIMEOUT = 30 # seconds From 2b4f321bba0e4d8fe86dd5fe5c295aa92070bb1e Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Tue, 15 Dec 2020 10:42:20 +0000 Subject: [PATCH 12/31] WIP: test batch api's --- test/e2e/e2e/tests.py | 42 +++++++++++++++++++++++++++++++- test/e2e/e2e/utils.py | 15 ++++++++++++ test/e2e/tests/aws/test_batch.py | 27 ++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 test/e2e/tests/aws/test_batch.py diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 6b8d5f3ba2..caf6e00148 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -20,7 +20,7 @@ import yaml from e2e.expectations import parse_expectations, assert_response_expectations -from e2e.utils import apis_ready, request_prediction +from e2e.utils import apis_ready, request_prediction, job_done TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" @@ -63,3 +63,43 @@ def test_realtime_api(client: cx.Client, api: str, timeout: int = None): assert_response_expectations(response, expectations["response"]) finally: delete_apis(client, api_names) + + +def test_batch_api( + client: cx.Client, api: str, deploy_timeout: int = None, job_timeout: int = None +): + api_dir = TEST_APIS_DIR / api + with open(str(api_dir / "cortex.yaml")) as f: + api_specs = yaml.safe_load(f) + + api_names = [api_spec["name"] for api_spec in api_specs] + for api_spec in api_specs: + client.create_api(api_spec=api_spec, project_dir=api_dir) + + try: + assert apis_ready( + client=client, api_names=api_names, timeout=deploy_timeout + ), f"apis {api_names} not ready" + + with open(str(api_dir / "sample.json")) as f: + payload = json.load(f) + + api_name = api_names[0] + response = request_prediction(client, api_name, payload) + + assert ( + response.status_code == HTTPStatus.OK + ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" + + job_spec = response.json() + + # monitor job progress + assert job_done( + client=client, + api_name=job_spec["api_name"], + job_id=job_spec["job_id"], + timeout=job_timeout, + ) + + finally: + delete_apis(client, api_names) diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index f9aad8e6aa..303a75e0fe 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -36,6 +36,21 @@ def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = count += 1 +def job_done(client: cx.Client, api_name: str, job_id: str, timeout: int = None): + count = 0 + while True: + if timeout is not None and count > timeout: + return False + + # FIXME + client.get_job(api_name, job_id) + if ready: + return True + + time.sleep(1) + count += 1 + + def request_prediction( client: cx.Client, api_name: str, payload: Union[List, Dict] ) -> requests.Response: diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py new file mode 100644 index 0000000000..c4e0c69719 --- /dev/null +++ b/test/e2e/tests/aws/test_batch.py @@ -0,0 +1,27 @@ +# Copyright 2020 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cortex as cx +import pytest + +import e2e.tests + +TIMEOUT_DEPLOY = 10 # seconds +TEST_APIS = ["batch/onnx", "batch/tensorflow", "batch/image-classifier"] + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS) +def test_batch_api(client: cx.Client, api: str): + e2e.tests.test_batch_api(client, api, timeout=TIMEOUT_DEPLOY) From 2a964d95aacf432605eb01f37e68a139ddb6f326 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Wed, 23 Dec 2020 12:44:49 +0000 Subject: [PATCH 13/31] Fix cortex client dir in e2e setup.py --- test/e2e/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/setup.py b/test/e2e/setup.py index 699d4a4ddf..3096d3b02c 100644 --- a/test/e2e/setup.py +++ b/test/e2e/setup.py @@ -17,7 +17,7 @@ from setuptools import setup, find_packages root = Path(__file__).parent.absolute() -cortex_client_dir = root.parent.parent / "pkg" / "workloads" / "cortex" / "client" +cortex_client_dir = root.parent.parent / "pkg" / "cortex" / "client" if not cortex_client_dir.exists(): raise ModuleNotFoundError(f"cortex client not found in {cortex_client_dir}") From c7f3dd6f036632c054b560cee872231b110993d2 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Wed, 23 Dec 2020 14:45:16 +0000 Subject: [PATCH 14/31] Skip tests instead of failing if flags are not passed in --- test/e2e/e2e/tests.py | 13 ++++++++++--- test/e2e/e2e/utils.py | 22 ++++++++++++++++++++++ test/e2e/pytest.ini | 2 +- test/e2e/tests/aws/conftest.py | 8 +++----- test/e2e/tests/aws/test_batch.py | 17 ++++++++++++++--- test/e2e/tests/conftest.py | 7 +++++++ test/e2e/tests/gcp/conftest.py | 8 +++----- 7 files changed, 60 insertions(+), 17 deletions(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index caf6e00148..625d8e2357 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import json from http import HTTPStatus from pathlib import Path @@ -20,7 +21,7 @@ import yaml from e2e.expectations import parse_expectations, assert_response_expectations -from e2e.utils import apis_ready, request_prediction, job_done +from e2e.utils import apis_ready, request_prediction, job_done, request_batch_prediction TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" @@ -66,7 +67,11 @@ def test_realtime_api(client: cx.Client, api: str, timeout: int = None): def test_batch_api( - client: cx.Client, api: str, deploy_timeout: int = None, job_timeout: int = None + client: cx.Client, + api: str, + test_bucket: str, + deploy_timeout: int = None, + job_timeout: int = None, ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / "cortex.yaml")) as f: @@ -85,7 +90,9 @@ def test_batch_api( payload = json.load(f) api_name = api_names[0] - response = request_prediction(client, api_name, payload) + response = request_batch_prediction( + client, api_name, item_list=payload, batch_size=2, config={"dest_s3_dir": test_bucket} + ) assert ( response.status_code == HTTPStatus.OK diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index 303a75e0fe..9ce0e89cf9 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -61,6 +61,28 @@ def request_prediction( return response +def request_batch_prediction( + client: cx.Client, + api_name: str, + item_list: List, + batch_size: int, + workers: int = 1, + config: Dict = None, +) -> requests.Response: + + api_info = client.get_api(api_name) + endpoint = api_info["endpoint"] + + batch_payload = { + "workers": workers, + "item_list": {"items": item_list, "batch_size": batch_size}, + "config": config, + } + response = requests.post(endpoint, json=batch_payload) + + return response + + def client_from_config(config_path: str) -> cx.Client: with open(config_path) as f: config = yaml.safe_load(f) diff --git a/test/e2e/pytest.ini b/test/e2e/pytest.ini index 620bae68f9..6b48448ef0 100644 --- a/test/e2e/pytest.ini +++ b/test/e2e/pytest.ini @@ -1,4 +1,4 @@ # pytest.ini [pytest] minversion = 6.0 -addopts = -s -v +addopts = -s -v -r sxf diff --git a/test/e2e/tests/aws/conftest.py b/test/e2e/tests/aws/conftest.py index 0921243ffb..881c28c85a 100644 --- a/test/e2e/tests/aws/conftest.py +++ b/test/e2e/tests/aws/conftest.py @@ -26,12 +26,10 @@ def client(request): return cx.client(env_name) config_path = request.config.getoption("--aws-config") - if config_path is None: - raise ValueError( - "missing arguments: --env-name or --aws-config " - ) + if config_path is not None: + return client_from_config(config_path) - return client_from_config(config_path) + pytest.skip("--aws-env or --aws-config must be passed to run aws tests") def pytest_configure(config): diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py index c4e0c69719..4538db754d 100644 --- a/test/e2e/tests/aws/test_batch.py +++ b/test/e2e/tests/aws/test_batch.py @@ -18,10 +18,21 @@ import e2e.tests TIMEOUT_DEPLOY = 10 # seconds -TEST_APIS = ["batch/onnx", "batch/tensorflow", "batch/image-classifier"] +JOB_TIMEOUT = 30 # seconds +TEST_APIS = ["batch/image-classifier"] + + +@pytest.fixture +def s3_bucket(request): + s3_bucket = request.config.getoption("--s3-bucket") + if not s3_bucket: + pytest.skip("--s3-bucket option is required to run batch tests") + + return s3_bucket @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) -def test_batch_api(client: cx.Client, api: str): - e2e.tests.test_batch_api(client, api, timeout=TIMEOUT_DEPLOY) +def test_batch_api(client: cx.Client, api: str, s3_bucket: str): + e2e.tests.test_batch_api(client, api, deploy_timeout=TIMEOUT_DEPLOY, job_timeout=JOB_TIMEOUT) + diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 8182d74b12..e51b3af7a1 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -38,3 +38,10 @@ def pytest_addoption(parser): default=None, help="set cortex GCP cluster config, to test on a new GCP cluster", ) + + parser.addoption( + "--s3-bucket", + action="store", + default=None, + help="set s3 bucket where batch jobs results will be stored", + ) diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py index 69b5d82eb7..785594b4c0 100644 --- a/test/e2e/tests/gcp/conftest.py +++ b/test/e2e/tests/gcp/conftest.py @@ -26,12 +26,10 @@ def client(request): return cx.client(env_name) config_path = request.config.getoption("--gcp-config") - if config_path is None: - raise ValueError( - "missing arguments: --env-name or --gcp-config " - ) + if config_path is not None: + return client_from_config(config_path) - return client_from_config(config_path) + pytest.skip("--gcp-env or --gcp-config must be passed to run gcp tests") def pytest_configure(config): From eb4c0b59dec182c0cb862fd0a34286cba0b615f5 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 24 Dec 2020 12:25:16 +0000 Subject: [PATCH 15/31] Fixed batch tests --- test/e2e/e2e/tests.py | 18 +++++++++++++----- test/e2e/e2e/utils.py | 22 +++++++++++++++++++--- test/e2e/tests/aws/test_batch.py | 7 ++++--- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 625d8e2357..26276e8ace 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -21,7 +21,13 @@ import yaml from e2e.expectations import parse_expectations, assert_response_expectations -from e2e.utils import apis_ready, request_prediction, job_done, request_batch_prediction +from e2e.utils import ( + apis_ready, + endpoint_ready, + request_prediction, + job_done, + request_batch_prediction, +) TEST_APIS_DIR = Path(__file__).parent.parent.parent / "apis" @@ -77,13 +83,15 @@ def test_batch_api( with open(str(api_dir / "cortex.yaml")) as f: api_specs = yaml.safe_load(f) + assert len(api_specs) == 1 + api_names = [api_spec["name"] for api_spec in api_specs] for api_spec in api_specs: client.create_api(api_spec=api_spec, project_dir=api_dir) try: - assert apis_ready( - client=client, api_names=api_names, timeout=deploy_timeout + assert endpoint_ready( + client=client, api_name=api_names[0], timeout=deploy_timeout ), f"apis {api_names} not ready" with open(str(api_dir / "sample.json")) as f: @@ -96,7 +104,7 @@ def test_batch_api( assert ( response.status_code == HTTPStatus.OK - ), f"status code: got {response.status_code}, expected {HTTPStatus.OK}" + ), f"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})" job_spec = response.json() @@ -106,7 +114,7 @@ def test_batch_api( api_name=job_spec["api_name"], job_id=job_spec["job_id"], timeout=job_timeout, - ) + ), f"job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})" finally: delete_apis(client, api_names) diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index 9ce0e89cf9..bd5a3daf81 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -13,6 +13,7 @@ # limitations under the License. import time +from http import HTTPStatus from typing import List, Optional, Dict, Union import cortex as cx @@ -36,15 +37,30 @@ def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = count += 1 +def endpoint_ready(client: cx.Client, api_name: str, timeout: int = None) -> bool: + count = 0 + while True: + if timeout is not None and count > timeout: + return False + + endpoint = client.get_api(api_name)["endpoint"] + response = requests.post(endpoint) + if response.status_code == HTTPStatus.BAD_REQUEST: + return True + + time.sleep(1) + count += 1 + + def job_done(client: cx.Client, api_name: str, job_id: str, timeout: int = None): count = 0 while True: if timeout is not None and count > timeout: return False - # FIXME - client.get_job(api_name, job_id) - if ready: + job_info = client.get_job(api_name, job_id) + done = job_info["job_status"]["status"] == "status_succeeded" + if done: return True time.sleep(1) diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py index 4538db754d..8db4d1d25d 100644 --- a/test/e2e/tests/aws/test_batch.py +++ b/test/e2e/tests/aws/test_batch.py @@ -18,7 +18,7 @@ import e2e.tests TIMEOUT_DEPLOY = 10 # seconds -JOB_TIMEOUT = 30 # seconds +JOB_TIMEOUT = 60 # seconds TEST_APIS = ["batch/image-classifier"] @@ -34,5 +34,6 @@ def s3_bucket(request): @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) def test_batch_api(client: cx.Client, api: str, s3_bucket: str): - e2e.tests.test_batch_api(client, api, deploy_timeout=TIMEOUT_DEPLOY, job_timeout=JOB_TIMEOUT) - + e2e.tests.test_batch_api( + client, api, test_bucket=s3_bucket, deploy_timeout=TIMEOUT_DEPLOY, job_timeout=JOB_TIMEOUT + ) From 0d999adee11d4dd312e87e1b2f781b7fee0efc49 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 24 Dec 2020 13:52:43 +0000 Subject: [PATCH 16/31] Add sample.json to onnx and tf batch examples --- test/apis/batch/onnx/sample.json | 3 +++ test/apis/batch/tensorflow/sample.json | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 test/apis/batch/onnx/sample.json create mode 100644 test/apis/batch/tensorflow/sample.json diff --git a/test/apis/batch/onnx/sample.json b/test/apis/batch/onnx/sample.json new file mode 100644 index 0000000000..eb45c463fd --- /dev/null +++ b/test/apis/batch/onnx/sample.json @@ -0,0 +1,3 @@ +[ + "https://i.imgur.com/PzXprwl.jpg" +] diff --git a/test/apis/batch/tensorflow/sample.json b/test/apis/batch/tensorflow/sample.json new file mode 100644 index 0000000000..eb45c463fd --- /dev/null +++ b/test/apis/batch/tensorflow/sample.json @@ -0,0 +1,3 @@ +[ + "https://i.imgur.com/PzXprwl.jpg" +] From 1ac903a0e367bda3b4f0e34ef065f7af22970767 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 24 Dec 2020 13:53:29 +0000 Subject: [PATCH 17/31] Add env vars to set tests behaviour and improved batch tests flakiness --- test/e2e/e2e/tests.py | 11 ++++-- test/e2e/e2e/utils.py | 51 +++++++++++---------------- test/e2e/tests/aws/test_batch.py | 22 ++++++++---- test/e2e/tests/aws/test_realtime.py | 3 +- test/e2e/tests/conftest.py | 6 ++++ test/e2e/tests/gcp/test_realtime.py | 3 +- test/e2e/tests/local/test_realtime.py | 3 +- 7 files changed, 57 insertions(+), 42 deletions(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 26276e8ace..9138d37682 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -78,6 +78,7 @@ def test_batch_api( test_bucket: str, deploy_timeout: int = None, job_timeout: int = None, + retry_attempts: int = 0, ): api_dir = TEST_APIS_DIR / api with open(str(api_dir / "cortex.yaml")) as f: @@ -98,9 +99,13 @@ def test_batch_api( payload = json.load(f) api_name = api_names[0] - response = request_batch_prediction( - client, api_name, item_list=payload, batch_size=2, config={"dest_s3_dir": test_bucket} - ) + + for i in range(retry_attempts + 1): + response = request_batch_prediction( + client, api_name, item_list=payload, batch_size=2, config={"dest_s3_dir": test_bucket} + ) + if response.status_code == HTTPStatus.OK: + break assert ( response.status_code == HTTPStatus.OK diff --git a/test/e2e/e2e/utils.py b/test/e2e/e2e/utils.py index bd5a3daf81..a21ea8d7f7 100644 --- a/test/e2e/e2e/utils.py +++ b/test/e2e/e2e/utils.py @@ -14,63 +14,55 @@ import time from http import HTTPStatus -from typing import List, Optional, Dict, Union +from typing import List, Optional, Dict, Union, Callable import cortex as cx import requests import yaml -def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = None) -> bool: - count = 0 +def wait_for(fn: Callable[[], bool], timeout=None) -> bool: + deadline = time.time() + timeout if timeout else None while True: - if timeout is not None and count > timeout: + if deadline is not None and time.time() > deadline: return False - ready = all( - [client.get_api(name)["status"]["status_code"] == "status_live" for name in api_names] - ) - if ready: + done = fn() + if done: return True time.sleep(1) - count += 1 -def endpoint_ready(client: cx.Client, api_name: str, timeout: int = None) -> bool: - count = 0 - while True: - if timeout is not None and count > timeout: - return False +def apis_ready(client: cx.Client, api_names: List[str], timeout: Optional[int] = None) -> bool: + def _is_ready(): + return all( + [client.get_api(name)["status"]["status_code"] == "status_live" for name in api_names] + ) + + return wait_for(_is_ready, timeout=timeout) + +def endpoint_ready(client: cx.Client, api_name: str, timeout: int = None) -> bool: + def _is_ready(): endpoint = client.get_api(api_name)["endpoint"] response = requests.post(endpoint) - if response.status_code == HTTPStatus.BAD_REQUEST: - return True + return response.status_code == HTTPStatus.BAD_REQUEST - time.sleep(1) - count += 1 + return wait_for(_is_ready, timeout=timeout) def job_done(client: cx.Client, api_name: str, job_id: str, timeout: int = None): - count = 0 - while True: - if timeout is not None and count > timeout: - return False - + def _is_ready(): job_info = client.get_job(api_name, job_id) - done = job_info["job_status"]["status"] == "status_succeeded" - if done: - return True + return job_info["job_status"]["status"] == "status_succeeded" - time.sleep(1) - count += 1 + return wait_for(_is_ready, timeout=timeout) def request_prediction( client: cx.Client, api_name: str, payload: Union[List, Dict] ) -> requests.Response: - api_info = client.get_api(api_name) response = requests.post(api_info["endpoint"], json=payload) @@ -85,7 +77,6 @@ def request_batch_prediction( workers: int = 1, config: Dict = None, ) -> requests.Response: - api_info = client.get_api(api_name) endpoint = api_info["endpoint"] diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py index 8db4d1d25d..faf37a0d5c 100644 --- a/test/e2e/tests/aws/test_batch.py +++ b/test/e2e/tests/aws/test_batch.py @@ -11,22 +11,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import cortex as cx import pytest import e2e.tests -TIMEOUT_DEPLOY = 10 # seconds -JOB_TIMEOUT = 60 # seconds -TEST_APIS = ["batch/image-classifier"] +DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)) # seconds +JOB_TIMEOUT = int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 120)) # seconds +TEST_APIS = ["batch/image-classifier", "batch/onnx", "batch/tensorflow"] @pytest.fixture def s3_bucket(request): - s3_bucket = request.config.getoption("--s3-bucket") + s3_bucket = os.environ.get("CORTEX_TEST_BATCH_S3_BUCKET_DIR") + s3_bucket = request.config.getoption("--s3-bucket") if s3_bucket is None else s3_bucket if not s3_bucket: - pytest.skip("--s3-bucket option is required to run batch tests") + pytest.skip( + "--s3-bucket option is required to run batch tests (alternatively set the " + "CORTEX_TEST_BATCH_S3_BUCKET_DIR env var) )" + ) return s3_bucket @@ -35,5 +40,10 @@ def s3_bucket(request): @pytest.mark.parametrize("api", TEST_APIS) def test_batch_api(client: cx.Client, api: str, s3_bucket: str): e2e.tests.test_batch_api( - client, api, test_bucket=s3_bucket, deploy_timeout=TIMEOUT_DEPLOY, job_timeout=JOB_TIMEOUT + client, + api, + test_bucket=s3_bucket, + deploy_timeout=DEPLOY_TIMEOUT, + job_timeout=JOB_TIMEOUT, + retry_attempts=5, ) diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py index 58c1e230da..fa6d635b95 100644 --- a/test/e2e/tests/aws/test_realtime.py +++ b/test/e2e/tests/aws/test_realtime.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import cortex as cx import pytest @@ -18,7 +19,7 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -DEPLOY_TIMEOUT = 30 # seconds +DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60)) # seconds @pytest.mark.usefixtures("client") diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index e51b3af7a1..80c73d5bab 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from dotenv import load_dotenv + def pytest_addoption(parser): parser.addoption( @@ -45,3 +47,7 @@ def pytest_addoption(parser): default=None, help="set s3 bucket where batch jobs results will be stored", ) + + +def pytest_configure(config): + load_dotenv(".env") diff --git a/test/e2e/tests/gcp/test_realtime.py b/test/e2e/tests/gcp/test_realtime.py index 3769ae84be..cb4efeacd8 100644 --- a/test/e2e/tests/gcp/test_realtime.py +++ b/test/e2e/tests/gcp/test_realtime.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import cortex as cx import pytest @@ -18,7 +19,7 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -DEPLOY_TIMEOUT = 30 # seconds +DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60)) # seconds @pytest.mark.usefixtures("client") diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py index 1cd565691b..b577113aa1 100644 --- a/test/e2e/tests/local/test_realtime.py +++ b/test/e2e/tests/local/test_realtime.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import cortex as cx import pytest @@ -18,7 +19,7 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -DEPLOY_TIMEOUT = 30 # seconds +DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60)) # seconds @pytest.fixture From 1d783460c531277c5c59012826b9d596be4a5671 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 24 Dec 2020 13:55:56 +0000 Subject: [PATCH 18/31] Fix linting issues --- test/e2e/e2e/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 9138d37682..c6db1acae4 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -102,7 +102,11 @@ def test_batch_api( for i in range(retry_attempts + 1): response = request_batch_prediction( - client, api_name, item_list=payload, batch_size=2, config={"dest_s3_dir": test_bucket} + client, + api_name, + item_list=payload, + batch_size=2, + config={"dest_s3_dir": test_bucket}, ) if response.status_code == HTTPStatus.OK: break From 59b08c3ba3b3e47fc836bb33bcc45721c783a597 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 28 Dec 2020 10:48:43 +0000 Subject: [PATCH 19/31] Update e2e setup.py with dependencies --- test/e2e/setup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/e2e/setup.py b/test/e2e/setup.py index 3096d3b02c..abcf017522 100644 --- a/test/e2e/setup.py +++ b/test/e2e/setup.py @@ -25,11 +25,17 @@ setup( name="e2e", version="master", - packages=find_packages(), - url="", + packages=find_packages(exclude=["tests"]), + url="https://github.com/cortexlabs/cortex", license="Apache 2.0 Licence", python_requires=">=3.6", - install_requires=["requests", "cortex"], + install_requires=[ + "requests==2.24.0", + "jsonschema==3.2.0", + "pytest>=6.1.2", + "python-dotenv==0.15.0", + "cortex", + ], dependency_links=[f"file://{cortex_client_dir}#egg=cortex"], author="Cortex Labs", author_email="hello@cortexlabs.com", From b85fbaa59904b715092d512f02dbfc9cf3e89dc1 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 28 Dec 2020 10:51:46 +0000 Subject: [PATCH 20/31] Update e2e setup.py --- test/e2e/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/setup.py b/test/e2e/setup.py index abcf017522..02c2b8eb62 100644 --- a/test/e2e/setup.py +++ b/test/e2e/setup.py @@ -27,7 +27,7 @@ version="master", packages=find_packages(exclude=["tests"]), url="https://github.com/cortexlabs/cortex", - license="Apache 2.0 Licence", + license="Apache License 2.0", python_requires=">=3.6", install_requires=[ "requests==2.24.0", @@ -38,6 +38,6 @@ ], dependency_links=[f"file://{cortex_client_dir}#egg=cortex"], author="Cortex Labs", - author_email="hello@cortexlabs.com", + author_email="dev@cortex.dev", description="Cortex E2E tests package", ) From c66dac9ef88e1fcb4980d60f3b52460cec961356 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 28 Dec 2020 11:23:55 +0000 Subject: [PATCH 21/31] Restructure tests README.md, add E2E tests README.md --- test/README.md | 70 +++------------------------------------------ test/apis/README.md | 67 +++++++++++++++++++++++++++++++++++++++++++ test/e2e/README.md | 68 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 66 deletions(-) create mode 100644 test/apis/README.md create mode 100644 test/e2e/README.md diff --git a/test/README.md b/test/README.md index 1eb711f57d..901d496d35 100644 --- a/test/README.md +++ b/test/README.md @@ -1,67 +1,5 @@ -# Examples +# Cortex Tests -## TensorFlow - -- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers. - -- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text. - -- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis. - -- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images. - -- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images. - -- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time. - -- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API. - -## Keras - -- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise. - -## PyTorch - -- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers. - -- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text. - -- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis. - -- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms. - -- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions. - -- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text. - -- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension. - -- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages. - -- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API. - -- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images. - -- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images. - -- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images. - -- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer. - -## ONNX - -- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers. - -- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset. - -- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API. - -## scikit-learn - -- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers. - -- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG. - -## spacy - -- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition. +- [Example APIs](apis/README.md) +- [End-to-end Tests](e2e/README.md) +- [Testing Utilities](utils/README.md) diff --git a/test/apis/README.md b/test/apis/README.md new file mode 100644 index 0000000000..1eb711f57d --- /dev/null +++ b/test/apis/README.md @@ -0,0 +1,67 @@ +# Examples + +## TensorFlow + +- [Iris classification](tensorflow/iris-classifier): deploy a model to classify iris flowers. + +- [Text generation](tensorflow/text-generator): deploy OpenAI's GPT-2 to generate text. + +- [Sentiment analysis](tensorflow/sentiment-analyzer): deploy a BERT model for sentiment analysis. + +- [Image classification](tensorflow/image-classifier-inception): deploy an Inception model to classify images. + +- [Image classification](tensorflow/image-classifier-resnet50): deploy a ResNet50 model to classify images. + +- [License plate reader](tensorflow/license-plate-reader): deploy a YOLOv3 model (and others) to identify license plates in real time. + +- [Multi-model classification](tensorflow/multi-model-classifier): deploy 3 models (ResNet50, Iris, Inception) in a single API. + +## Keras + +- [Denoisify text documents](keras/document-denoiser): deploy an Autoencoder model to clean text document images of noise. + +## PyTorch + +- [Iris classification](pytorch/iris-classifier): deploy a model to classify iris flowers. + +- [Text generation](pytorch/text-generator): deploy Hugging Face's GPT-2 model to generate text. + +- [Sentiment analysis](pytorch/sentiment-analyzer): deploy a Hugging Face transformers model for sentiment analysis. + +- [Search completion](pytorch/search-completer): deploy a Facebook's RoBERTa model to complete search terms. + +- [Answer generation](pytorch/answer-generator): deploy Microsoft's DialoGPT model to answer questions. + +- [Text summarization](pytorch/text-summarizer): deploy a BART model (from Hugging Face's transformers library) to summarize text. + +- [Reading comprehension](pytorch/reading-comprehender): deploy an AllenNLP model for reading comprehension. + +- [Language identification](pytorch/language-identifier): deploy a fastText model to identify languages. + +- [Multi-model text analysis](pytorch/multi-model-text-analyzer): deploy 2 models (Sentiment and Summarization analyzers) in a single API. + +- [Image classification](pytorch/image-classifier-alexnet): deploy an AlexNet model from TorchVision to classify images. + +- [Image classification](pytorch/image-classifier-resnet50): deploy a ResNet50 model from TorchVision to classify images. + +- [Object detection](pytorch/object-detector): deploy a Faster R-CNN model from TorchVision to detect objects in images. + +- [Question generator](pytorch/question-generator): deploy a transformers model to generate questions given text and the correct answer. + +## ONNX + +- [Iris classification](onnx/iris-classifier): deploy an XGBoost model (exported in ONNX) to classify iris flowers. + +- [YOLOv5 YouTube detection](onnx/yolov5-youtube): deploy a YOLOv5 model trained on COCO val2017 dataset. + +- [Multi-model classification](onnx/multi-model-classifier): deploy 3 models (ResNet50, MobileNet, ShuffleNet) in a single API. + +## scikit-learn + +- [Iris classification](sklearn/iris-classifier): deploy a model to classify iris flowers. + +- [MPG estimation](sklearn/mpg-estimator): deploy a linear regression model to estimate MPG. + +## spacy + +- [Entity recognizer](spacy/entity-recognizer): deploy a spacy model for named entity recognition. diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000000..fba3005115 --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,68 @@ +# End-to-end Tests + +## Dependencies + +Install the `e2e` package, from the project directory: + +```shell +pip install -e test/e2e +``` + +## Running the tests + +### AWS + +From an existing cluster: + +```shell +pytest test/e2e/tests/aws --aws-env +``` + +Using a new cluster, created for testing only and deleted afterwards: + +```shell +pytest test/e2e/tests/aws --aws-config +``` + +**Note:** For the BatchAPI tests, the `--s3-bucket` option should be provided with an +AWS S3 bucket for testing purposes. It is more convinient however to define +this bucket through an environment variable, see [configuration](#configuration). + +### GCP + +From an existing cluster: + +```shell +pytest test/e2e/tests/gcp --gcp-env +``` + +Using a new cluster, created for testing only and deleted afterwards: + +```shell +pytest test/e2e/tests/aws --gcp-config +``` + +### All Tests + +You can run all tests at once, by running them from the parent +directory `test/e2e/tests`. The provider specific options should be provided +accordingly, or the test cases will be skipped. + +e.g. + +```shell +pytest test/e2e/tests --aws-env --gcp-env +``` + +## Configuration + +It is possible to configure the behaviour of the tests by defining +environment variables or a `.env` file at the project directory. + +```dotenv +# .env file +CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT=60 +CORTEX_TEST_BATCH_DEPLOY_TIMEOUT=30 +CORTEX_TEST_BATCH_JOB_TIMEOUT=120 +CORTEX_TEST_BATCH_S3_BUCKET_DIR=s3:///test/jobs +``` From df45223187c985cd629231ac7292847ba546c358 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 28 Dec 2020 11:24:34 +0000 Subject: [PATCH 22/31] Add sleep before request retry --- test/e2e/e2e/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index c6db1acae4..78394ca4c4 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -13,6 +13,7 @@ # limitations under the License. import json +import time from http import HTTPStatus from pathlib import Path from typing import List @@ -111,6 +112,8 @@ def test_batch_api( if response.status_code == HTTPStatus.OK: break + time.sleep(1) + assert ( response.status_code == HTTPStatus.OK ), f"status code: got {response.status_code}, expected {HTTPStatus.OK} ({response.text})" From 52b710fbee3114984997d5579b1f4f056b7ff839 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 28 Dec 2020 15:26:18 +0000 Subject: [PATCH 23/31] Refactor tests configuration --- test/e2e/README.md | 11 ++++--- test/e2e/tests/aws/conftest.py | 6 ++-- test/e2e/tests/aws/test_batch.py | 23 +++++---------- test/e2e/tests/aws/test_realtime.py | 9 +++--- test/e2e/tests/conftest.py | 42 ++++++++++++++++++++++++++- test/e2e/tests/gcp/conftest.py | 7 +++-- test/e2e/tests/gcp/test_realtime.py | 9 +++--- test/e2e/tests/local/test_realtime.py | 9 +++--- 8 files changed, 76 insertions(+), 40 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index fba3005115..ca50decb21 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -15,13 +15,13 @@ pip install -e test/e2e From an existing cluster: ```shell -pytest test/e2e/tests/aws --aws-env +pytest test/e2e/tests -k aws --aws-env ``` Using a new cluster, created for testing only and deleted afterwards: ```shell -pytest test/e2e/tests/aws --aws-config +pytest test/e2e/tests -k aws --aws-config ``` **Note:** For the BatchAPI tests, the `--s3-bucket` option should be provided with an @@ -33,19 +33,18 @@ this bucket through an environment variable, see [configuration](#configuration) From an existing cluster: ```shell -pytest test/e2e/tests/gcp --gcp-env +pytest test/e2e/tests -k gcp --gcp-env ``` Using a new cluster, created for testing only and deleted afterwards: ```shell -pytest test/e2e/tests/aws --gcp-config +pytest test/e2e/tests -k gcp --gcp-config ``` ### All Tests -You can run all tests at once, by running them from the parent -directory `test/e2e/tests`. The provider specific options should be provided +You can run all tests at once, however the provider specific options should be passed accordingly, or the test cases will be skipped. e.g. diff --git a/test/e2e/tests/aws/conftest.py b/test/e2e/tests/aws/conftest.py index 881c28c85a..90a7c65744 100644 --- a/test/e2e/tests/aws/conftest.py +++ b/test/e2e/tests/aws/conftest.py @@ -20,12 +20,12 @@ @pytest.fixture -def client(request): - env_name = request.config.getoption("--aws-env") +def client(config): + env_name = config["aws"]["env"] if env_name: return cx.client(env_name) - config_path = request.config.getoption("--aws-config") + config_path = config["aws"]["config"] if config_path is not None: return client_from_config(config_path) diff --git a/test/e2e/tests/aws/test_batch.py b/test/e2e/tests/aws/test_batch.py index faf37a0d5c..8599f7cf3c 100644 --- a/test/e2e/tests/aws/test_batch.py +++ b/test/e2e/tests/aws/test_batch.py @@ -11,39 +11,32 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os + +from typing import Dict import cortex as cx import pytest import e2e.tests -DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)) # seconds -JOB_TIMEOUT = int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 120)) # seconds TEST_APIS = ["batch/image-classifier", "batch/onnx", "batch/tensorflow"] -@pytest.fixture -def s3_bucket(request): - s3_bucket = os.environ.get("CORTEX_TEST_BATCH_S3_BUCKET_DIR") - s3_bucket = request.config.getoption("--s3-bucket") if s3_bucket is None else s3_bucket +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS) +def test_batch_api(config: Dict, client: cx.Client, api: str): + s3_bucket = config["aws"].get("s3_bucket") if not s3_bucket: pytest.skip( "--s3-bucket option is required to run batch tests (alternatively set the " "CORTEX_TEST_BATCH_S3_BUCKET_DIR env var) )" ) - return s3_bucket - - -@pytest.mark.usefixtures("client") -@pytest.mark.parametrize("api", TEST_APIS) -def test_batch_api(client: cx.Client, api: str, s3_bucket: str): e2e.tests.test_batch_api( client, api, test_bucket=s3_bucket, - deploy_timeout=DEPLOY_TIMEOUT, - job_timeout=JOB_TIMEOUT, + deploy_timeout=config["global"]["batch_deploy_timeout"], + job_timeout=config["global"]["batch_job_timeout"], retry_attempts=5, ) diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py index fa6d635b95..f9d1274175 100644 --- a/test/e2e/tests/aws/test_realtime.py +++ b/test/e2e/tests/aws/test_realtime.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os +from typing import Dict import cortex as cx import pytest @@ -19,10 +19,11 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60)) # seconds @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) -def test_realtime_api(client: cx.Client, api: str): - e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) +def test_realtime_api(config: Dict, client: cx.Client, api: str): + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + ) diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 80c73d5bab..946108eb3f 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import pytest +import yaml from dotenv import load_dotenv @@ -40,7 +43,6 @@ def pytest_addoption(parser): default=None, help="set cortex GCP cluster config, to test on a new GCP cluster", ) - parser.addoption( "--s3-bucket", action="store", @@ -51,3 +53,41 @@ def pytest_addoption(parser): def pytest_configure(config): load_dotenv(".env") + + s3_bucket = os.environ.get("CORTEX_TEST_BATCH_S3_BUCKET_DIR") + s3_bucket = config.getoption("--s3-bucket") if s3_bucket is None else s3_bucket + + configuration = { + "aws": { + "env": config.getoption("--aws-env"), + "config": config.getoption("--aws-config"), + "s3_bucket": s3_bucket, + }, + "gcp": { + "env": config.getoption("--gcp-env"), + "config": config.getoption("--gcp-config"), + }, + "global": { + "realtime_deploy_timeout": int( + os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60) + ), + "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), + "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 120)), + }, + } + + class Config: + @pytest.fixture(autouse=True) + def config(self): + return configuration + + config.pluginmanager.register(Config()) + + print("\n----- Test Configuration -----\n") + print(yaml.dump(configuration, indent=2)) + + if configuration["aws"]["env"] and configuration["aws"]["config"]: + raise ValueError("--aws-env and --aws-config are mutually exclusive") + + if configuration["gcp"]["env"] and configuration["gcp"]["config"]: + raise ValueError("--gcp-env and --gcp-config are mutually exclusive") diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py index 785594b4c0..a18c9ee281 100644 --- a/test/e2e/tests/gcp/conftest.py +++ b/test/e2e/tests/gcp/conftest.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Dict import cortex as cx import pytest @@ -20,12 +21,12 @@ @pytest.fixture -def client(request): - env_name = request.config.getoption("--gcp-env") +def client(config: Dict): + env_name = config["gcp"]["env"] if env_name: return cx.client(env_name) - config_path = request.config.getoption("--gcp-config") + config_path = config["gcp"]["config"] if config_path is not None: return client_from_config(config_path) diff --git a/test/e2e/tests/gcp/test_realtime.py b/test/e2e/tests/gcp/test_realtime.py index cb4efeacd8..4062ca30ed 100644 --- a/test/e2e/tests/gcp/test_realtime.py +++ b/test/e2e/tests/gcp/test_realtime.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os +from typing import Dict import cortex as cx import pytest @@ -19,10 +19,11 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60)) # seconds @pytest.mark.usefixtures("client") @pytest.mark.parametrize("api", TEST_APIS) -def test_realtime_apis(client: cx.Client, api: str): - e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) +def test_realtime_apis(config: Dict, client: cx.Client, api: str): + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + ) diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py index b577113aa1..a61708a648 100644 --- a/test/e2e/tests/local/test_realtime.py +++ b/test/e2e/tests/local/test_realtime.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os +from typing import Dict import cortex as cx import pytest @@ -19,7 +19,6 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] -DEPLOY_TIMEOUT = int(os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60)) # seconds @pytest.fixture @@ -28,5 +27,7 @@ def client(): @pytest.mark.parametrize("api", TEST_APIS) -def test_realtime_apis(client: cx.Client, api: str): - e2e.tests.test_realtime_api(client=client, api=api, timeout=DEPLOY_TIMEOUT) +def test_realtime_apis(config: Dict, client: cx.Client, api: str): + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + ) From 59ce55db3ebc0ceba70caed9a7602f3e21258c7d Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Mon, 28 Dec 2020 15:29:22 +0000 Subject: [PATCH 24/31] Fix linting errors --- test/e2e/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index ca50decb21..8ba85dcb82 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -24,8 +24,8 @@ Using a new cluster, created for testing only and deleted afterwards: pytest test/e2e/tests -k aws --aws-config ``` -**Note:** For the BatchAPI tests, the `--s3-bucket` option should be provided with an -AWS S3 bucket for testing purposes. It is more convinient however to define +**Note:** For the BatchAPI tests, the `--s3-bucket` option should be provided with an +AWS S3 bucket for testing purposes. It is more convinient however to define this bucket through an environment variable, see [configuration](#configuration). ### GCP @@ -44,7 +44,7 @@ pytest test/e2e/tests -k gcp --gcp-config ### All Tests -You can run all tests at once, however the provider specific options should be passed +You can run all tests at once, however the provider specific options should be passed accordingly, or the test cases will be skipped. e.g. @@ -55,7 +55,7 @@ pytest test/e2e/tests --aws-env --gcp-env ## Configuration -It is possible to configure the behaviour of the tests by defining +It is possible to configure the behaviour of the tests by defining environment variables or a `.env` file at the project directory. ```dotenv From ac67ea67250510b7fc650c11b1830722d412b286 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Wed, 30 Dec 2020 11:53:39 +0000 Subject: [PATCH 25/31] Address PR comments --- test/README.md | 6 +++--- test/e2e/e2e/expectations.py | 2 +- test/e2e/e2e/tests.py | 22 +++++++++------------- test/e2e/setup.py | 1 + test/e2e/tests/conftest.py | 4 ++-- test/e2e/tests/gcp/conftest.py | 12 ++++++------ 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/test/README.md b/test/README.md index 901d496d35..680a154449 100644 --- a/test/README.md +++ b/test/README.md @@ -1,5 +1,5 @@ # Cortex Tests -- [Example APIs](apis/README.md) -- [End-to-end Tests](e2e/README.md) -- [Testing Utilities](utils/README.md) +- [Example APIs](apis) +- [End-to-end Tests](e2e) +- [Testing Utilities](utils) diff --git a/test/e2e/e2e/expectations.py b/test/e2e/e2e/expectations.py index 500723e047..ca88d061bf 100644 --- a/test/e2e/e2e/expectations.py +++ b/test/e2e/e2e/expectations.py @@ -31,7 +31,7 @@ def assert_response_expectations(response: requests.Response, expectations: Dict expected = expectations.get("expected") if expected: output = _get_response_content(response, content_type) - assert output == expected, f"expected response: got {output}, expected {expected}" + assert output == expected, f"unexpected response: got {output}, expected {expected}" expected_json_schema = expectations.get("json_schema") if expected_json_schema: diff --git a/test/e2e/e2e/tests.py b/test/e2e/e2e/tests.py index 78394ca4c4..cac1c44bce 100644 --- a/test/e2e/e2e/tests.py +++ b/test/e2e/e2e/tests.py @@ -48,19 +48,18 @@ def test_realtime_api(client: cx.Client, api: str, timeout: int = None): if expectations_file.exists(): expectations = parse_expectations(str(expectations_file)) - api_names = [api_spec["name"] for api_spec in api_specs] + api_name = api_specs[0]["name"] for api_spec in api_specs: client.create_api(api_spec=api_spec, project_dir=api_dir) try: assert apis_ready( - client=client, api_names=api_names, timeout=timeout - ), f"apis {api_names} not ready" + client=client, api_names=[api_name], timeout=timeout + ), f"apis {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) - api_name = api_names[0] response = request_prediction(client, api_name, payload) assert ( @@ -70,7 +69,7 @@ def test_realtime_api(client: cx.Client, api: str, timeout: int = None): if expectations and "response" in expectations: assert_response_expectations(response, expectations["response"]) finally: - delete_apis(client, api_names) + delete_apis(client, [api_name]) def test_batch_api( @@ -87,20 +86,17 @@ def test_batch_api( assert len(api_specs) == 1 - api_names = [api_spec["name"] for api_spec in api_specs] - for api_spec in api_specs: - client.create_api(api_spec=api_spec, project_dir=api_dir) + api_name = api_specs[0]["name"] + client.create_api(api_spec=api_specs[0], project_dir=api_dir) try: assert endpoint_ready( - client=client, api_name=api_names[0], timeout=deploy_timeout - ), f"apis {api_names} not ready" + client=client, api_name=api_name, timeout=deploy_timeout + ), f"api {api_name} not ready" with open(str(api_dir / "sample.json")) as f: payload = json.load(f) - api_name = api_names[0] - for i in range(retry_attempts + 1): response = request_batch_prediction( client, @@ -129,4 +125,4 @@ def test_batch_api( ), f"job did not succeed (api_name: {api_name}, job_id: {job_spec['job_id']})" finally: - delete_apis(client, api_names) + delete_apis(client, [api_name]) diff --git a/test/e2e/setup.py b/test/e2e/setup.py index 02c2b8eb62..1bf6a6a9aa 100644 --- a/test/e2e/setup.py +++ b/test/e2e/setup.py @@ -34,6 +34,7 @@ "jsonschema==3.2.0", "pytest>=6.1.2", "python-dotenv==0.15.0", + "pyyaml>=5.3.1", "cortex", ], dependency_links=[f"file://{cortex_client_dir}#egg=cortex"], diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 946108eb3f..b8515c93ea 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -55,7 +55,7 @@ def pytest_configure(config): load_dotenv(".env") s3_bucket = os.environ.get("CORTEX_TEST_BATCH_S3_BUCKET_DIR") - s3_bucket = config.getoption("--s3-bucket") if s3_bucket is None else s3_bucket + s3_bucket = config.getoption("--s3-bucket") if not s3_bucket else s3_bucket configuration = { "aws": { @@ -69,7 +69,7 @@ def pytest_configure(config): }, "global": { "realtime_deploy_timeout": int( - os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 60) + os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 120) ), "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 120)), diff --git a/test/e2e/tests/gcp/conftest.py b/test/e2e/tests/gcp/conftest.py index a18c9ee281..58fde4be81 100644 --- a/test/e2e/tests/gcp/conftest.py +++ b/test/e2e/tests/gcp/conftest.py @@ -34,12 +34,12 @@ def client(config: Dict): def pytest_configure(config): - aws_config = config.getoption("--gcp-config") - if aws_config: - e2e.create_cluster(aws_config) + gcp_config = config.getoption("--gcp-config") + if gcp_config: + e2e.create_cluster(gcp_config) def pytest_unconfigure(config): - aws_config = config.getoption("--gcp-config") - if aws_config: - e2e.delete_cluster(aws_config) + gcp_config = config.getoption("--gcp-config") + if gcp_config: + e2e.delete_cluster(gcp_config) From c644abb3a744c57aea6e17cf6b73fcc54d8ccb9a Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Wed, 30 Dec 2020 11:53:49 +0000 Subject: [PATCH 26/31] Remove local tests --- test/e2e/tests/local/__init__.py | 13 ----------- test/e2e/tests/local/test_realtime.py | 33 --------------------------- 2 files changed, 46 deletions(-) delete mode 100644 test/e2e/tests/local/__init__.py delete mode 100644 test/e2e/tests/local/test_realtime.py diff --git a/test/e2e/tests/local/__init__.py b/test/e2e/tests/local/__init__.py deleted file mode 100644 index ab054358f3..0000000000 --- a/test/e2e/tests/local/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2020 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/test/e2e/tests/local/test_realtime.py b/test/e2e/tests/local/test_realtime.py deleted file mode 100644 index a61708a648..0000000000 --- a/test/e2e/tests/local/test_realtime.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2020 Cortex Labs, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Dict - -import cortex as cx -import pytest - -import e2e.tests - -TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] - - -@pytest.fixture -def client(): - return cx.client("local") - - -@pytest.mark.parametrize("api", TEST_APIS) -def test_realtime_apis(config: Dict, client: cx.Client, api: str): - e2e.tests.test_realtime_api( - client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] - ) From 9bffb5f850ef281c7bb3ec480c8276484917a9f9 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 30 Dec 2020 09:25:21 -0800 Subject: [PATCH 27/31] Update pytest version --- test/e2e/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/setup.py b/test/e2e/setup.py index 1bf6a6a9aa..4ad6789a2d 100644 --- a/test/e2e/setup.py +++ b/test/e2e/setup.py @@ -32,7 +32,7 @@ install_requires=[ "requests==2.24.0", "jsonschema==3.2.0", - "pytest>=6.1.2", + "pytest==6.1.*", "python-dotenv==0.15.0", "pyyaml>=5.3.1", "cortex", From 9993471807663a37ef49444e905d7df666b5b075 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 30 Dec 2020 09:26:36 -0800 Subject: [PATCH 28/31] Update README.md --- test/e2e/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e/README.md b/test/e2e/README.md index 8ba85dcb82..d6a0eeba2a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -8,8 +8,16 @@ Install the `e2e` package, from the project directory: pip install -e test/e2e ``` +This only needs to be installed once (not on every code change). + ## Running the tests +Before running tests, instruct the Python client to use your development CLI binary: + +```shell +export CORTEX_CLI_PATH=~/src/github.com/cortexlabs/cortex/bin/cortex +``` + ### AWS From an existing cluster: From 91e38db4aebce1a817f282b98288f9baee903e28 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 30 Dec 2020 09:28:08 -0800 Subject: [PATCH 29/31] Update CONTRIBUTING.md --- CONTRIBUTING.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5bfb263ae0..9885b3b51b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -228,7 +228,11 @@ export CORTEX_DEV_DEFAULT_PREDICTOR_IMAGE_REGISTRY="cortexlabs" export CORTEX_TELEMETRY_SENTRY_DSN="https://c334df915c014ffa93f2076769e5b334@sentry.io/1848098" export CORTEX_TELEMETRY_SEGMENT_WRITE_KEY="0WvoJyCey9z1W2EW7rYTPJUMRYat46dl" -alias cortex='$HOME/bin/cortex' # your path may be different depending on where you cloned the repo +# instruct the Python client to use your development CLI binary (update the path to point to your cortex repo) +export CORTEX_CLI_PATH="/bin/cortex" + +# create a cortex alias which runs your development CLI +alias cortex="$CORTEX_CLI_PATH" ``` Refresh your bash profile: From 724e25aaa247394b9ab9cbca9e652214ff61d50c Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 30 Dec 2020 09:30:08 -0800 Subject: [PATCH 30/31] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2dc8f56e65..f98ece8f4c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .python-version .env .venv +*.egg-info # OSX .DS_Store From 62880695f701e529017989ba8f31154782b8ff62 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Wed, 30 Dec 2020 18:57:21 +0000 Subject: [PATCH 31/31] Fix e2e test README.md --- test/e2e/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index d6a0eeba2a..3edf57d2d1 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -15,7 +15,7 @@ This only needs to be installed once (not on every code change). Before running tests, instruct the Python client to use your development CLI binary: ```shell -export CORTEX_CLI_PATH=~/src/github.com/cortexlabs/cortex/bin/cortex +export CORTEX_CLI_PATH=/bin/cortex ``` ### AWS