Improve the coverage of the e2e test suite

.circleci/config.yml

@@ -58,7 +58,10 @@ commands:
           command: cortex cluster up << parameters.config >> --configure-env aws -y
       - run:
           name: Run E2E Tests
-          command: pytest -v test/e2e/tests --env aws
+          command: |
+            pytest -v test/e2e/tests --env aws --skip-autoscaling --skip-load --skip-long-running
+            pytest -v test/e2e/tests --env aws -k test_autoscaling
+            pytest -v test/e2e/tests --env aws -k test_load
       - run:
           name: Delete Cluster
           command: cortex cluster down --config << parameters.config >> -y
@@ -137,8 +140,8 @@ jobs:
             node_groups:
               - name: spot
                 instance_type: t3.medium
-                min_instances: 0
-                max_instances: 1
+                min_instances: 10
+                max_instances: 10
                 spot: true
               - name: cpu
                 instance_type: c5.xlarge

pkg/cortex/serve/cortex_internal/lib/api/async_api.py

@@ -27,7 +27,6 @@
 from cortex_internal.lib.client.tensorflow import TensorFlowClient
 from cortex_internal.lib.exceptions import CortexException, UserException, UserRuntimeException
 from cortex_internal.lib.metrics import MetricsClient
-from cortex_internal.lib.model import ModelsHolder
 from cortex_internal.lib.storage import S3
 from cortex_internal.lib.type import (
     predictor_type_from_api_spec,

test/apis/batch/sum/cortex.yaml

@@ -0,0 +1,8 @@
+- name: summing-api
+  kind: BatchAPI
+  predictor:
+    type: python
+    path: predictor.py
+  compute:
+    cpu: 100m
+    mem: 200Mi

test/apis/batch/sum/predictor.py

@@ -0,0 +1,24 @@
+import os
+import boto3
+import json
+import re
+
+
+class PythonPredictor:
+    def __init__(self, config, job_spec):
+        if len(config.get("dest_s3_dir", "")) == 0:
+            raise Exception("'dest_s3_dir' field was not provided in job submission")
+
+        self.s3 = boto3.client("s3")
+
+        self.bucket, self.key = re.match("s3://(.+?)/(.+)", config["dest_s3_dir"]).groups()
+        self.key = os.path.join(self.key, job_spec["job_id"])
+        self.list = []
+
+    def predict(self, payload, batch_id):
+        for numbers_list in payload:
+            self.list.append(sum(numbers_list))
+
+    def on_job_complete(self):
+        json_output = json.dumps(self.list)
+        self.s3.put_object(Bucket=self.bucket, Key=f"{self.key}.json", Body=json_output)

test/apis/batch/sum/sample.json

@@ -0,0 +1,3 @@
+[
+    [1, 2, 67, -2, 43]
+]

test/apis/batch/sum/sample_generator.py

@@ -0,0 +1,9 @@
+from typing import List
+from random import sample
+
+RANGE = 10 ** 12
+LENGTH = 5
+
+
+def generate_sample() -> List[int]:
+    return sample(range(RANGE), LENGTH)

test/apis/sleep/cortex.yaml

@@ -3,3 +3,5 @@
   predictor:
     type: python
     path: predictor.py
+  compute:
+    cpu: 100m

test/apis/tensorflow/iris-classifier/cortex.yaml

@@ -5,3 +5,5 @@
     path: predictor.py
     models:
       path: s3://cortex-examples/tensorflow/iris-classifier/nn/
+  compute:
+    mem: 250Mi

test/e2e/README.md

@@ -44,6 +44,18 @@ It is possible to skip GPU tests by passing the `--skip-gpus` flag to the pytest
 
 It is possible to skip Inferentia tests by passing the `--skip-infs` flag to the pytest command.
 
+### Skip Autoscaling Test
+
+It is possible to skip the autoscaling test by passing the `--skip-autoscaling` flag to the pytest command.
+
+### Skip Load Test
+
+It is possible to skip the load tests by passing the `--skip-load` flag to the pytest command.
+
+### Skip Long Running Test
+
+It is possible to skip the long running test by passing the `--skip-long-running` flag to the pytest command.
+
 ## Configuration
 
 It is possible to configure the behaviour of the tests by defining environment variables or a `.env` file at the project

test/e2e/e2e/exceptions.py

@@ -8,3 +8,7 @@ class ClusterDeletionException(Exception):
 
 class ExpectationsValidationException(Exception):
     pass
+
+
+class GeneratorValidationException(Exception):
+    pass

test/e2e/e2e/generator.py

@@ -0,0 +1,48 @@
+# Copyright 2021 Cortex Labs, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import pathlib
+from typing import Any, Callable, List
+import sys
+import inspect
+
+from e2e.exceptions import GeneratorValidationException
+
+
+def load_generator(sample_generator: pathlib.Path) -> Callable[[], List[int]]:
+    api_dir = str(sample_generator.parent)
+
+    sys.path.append(api_dir)
+    sample_generator_module = importlib.import_module(str(pathlib.Path(sample_generator).stem))
+    sys.path.pop()
+
+    validate_module(sample_generator_module)
+
+    return sample_generator_module.generate_sample
+
+
+def validate_module(sample_generator_module: Any):
+    if not hasattr(sample_generator_module, "generate_sample"):
+        raise GeneratorValidationException(
+            "sample generator module doesn't have a function called 'generate_sample'"
+        )
+
+    if not inspect.isfunction(getattr(sample_generator_module, "generate_sample")):
+        raise GeneratorValidationException("'generate_sample' is not a function")
+
+    if inspect.getfullargspec(getattr(sample_generator_module, "generate_sample")).args != []:
+        raise GeneratorValidationException(
+            "'generate_sample' function must not have any parameters"
+        )