From 49435aebc627df43cee892c80417b708cc512765 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 11 Feb 2021 19:36:13 +0100 Subject: [PATCH 1/5] WIP: GPU tests --- test/e2e/tests/aws/test_realtime.py | 13 +++++++++++++ test/e2e/tests/conftest.py | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py index 5a5f2db31e..6875b143b4 100644 --- a/test/e2e/tests/aws/test_realtime.py +++ b/test/e2e/tests/aws/test_realtime.py @@ -19,6 +19,7 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] +TEST_APIS_GPU = ["pytorch/text-generator", "tensorflow/text-generator"] @pytest.mark.usefixtures("client") @@ -27,3 +28,15 @@ def test_realtime_api(config: Dict, client: cx.Client, api: str): e2e.tests.test_realtime_api( client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] ) + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS_GPU) +def test_realtime_api_gpu(config: Dict, client: cx.Client, api: str): + skip_gpus = config["global"].get("skip_gpus", False) + if not skip_gpus: + pytest.skip("--skip-gpus flag detected, skipping GPU tests") + + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + ) diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 5c6068aca6..6e5462c397 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -49,6 +49,11 @@ def pytest_addoption(parser): default=None, help="set s3 path where batch jobs results will be stored", ) + parser.addoption( + "--skip-gpus", + action="store_true", + help="skip GPU tests", + ) def pytest_configure(config): @@ -73,6 +78,7 @@ def pytest_configure(config): ), "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 200)), + "skip_gpus": config.getoption("--skip-gpus") }, } From 59c950204a8623862d9fb5ae998127ba9b63c90f Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Thu, 11 Feb 2021 19:53:01 +0100 Subject: [PATCH 2/5] Fix skip flag and add GPU tests to GCP --- test/e2e/tests/aws/test_realtime.py | 2 +- test/e2e/tests/gcp/test_realtime.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/test/e2e/tests/aws/test_realtime.py b/test/e2e/tests/aws/test_realtime.py index 6875b143b4..af334697f0 100644 --- a/test/e2e/tests/aws/test_realtime.py +++ b/test/e2e/tests/aws/test_realtime.py @@ -34,7 +34,7 @@ def test_realtime_api(config: Dict, client: cx.Client, api: str): @pytest.mark.parametrize("api", TEST_APIS_GPU) def test_realtime_api_gpu(config: Dict, client: cx.Client, api: str): skip_gpus = config["global"].get("skip_gpus", False) - if not skip_gpus: + if skip_gpus: pytest.skip("--skip-gpus flag detected, skipping GPU tests") e2e.tests.test_realtime_api( diff --git a/test/e2e/tests/gcp/test_realtime.py b/test/e2e/tests/gcp/test_realtime.py index cf44310994..07b5644fca 100644 --- a/test/e2e/tests/gcp/test_realtime.py +++ b/test/e2e/tests/gcp/test_realtime.py @@ -19,6 +19,7 @@ import e2e.tests TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"] +TEST_APIS_GPU = ["pytorch/text-generator", "tensorflow/text-generator"] @pytest.mark.usefixtures("client") @@ -27,3 +28,15 @@ def test_realtime_apis(config: Dict, client: cx.Client, api: str): e2e.tests.test_realtime_api( client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] ) + + +@pytest.mark.usefixtures("client") +@pytest.mark.parametrize("api", TEST_APIS_GPU) +def test_realtime_api_gpu(config: Dict, client: cx.Client, api: str): + skip_gpus = config["global"].get("skip_gpus", False) + if skip_gpus: + pytest.skip("--skip-gpus flag detected, skipping GPU tests") + + e2e.tests.test_realtime_api( + client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"] + ) From 41b76461e2546a3b28d8ebc0152d595033936f7f Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 12 Feb 2021 12:27:19 +0100 Subject: [PATCH 3/5] Update timeout for realtime deployment --- test/e2e/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index 6e5462c397..e97345aed4 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -74,7 +74,7 @@ def pytest_configure(config): }, "global": { "realtime_deploy_timeout": int( - os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 120) + os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 200) ), "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 200)), From 44a77b8b8662c6452f0e036dd91a7cda90d94f86 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 12 Feb 2021 12:31:54 +0100 Subject: [PATCH 4/5] Fix linting errors --- test/e2e/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/tests/conftest.py b/test/e2e/tests/conftest.py index e97345aed4..5bf2fa10e3 100644 --- a/test/e2e/tests/conftest.py +++ b/test/e2e/tests/conftest.py @@ -78,7 +78,7 @@ def pytest_configure(config): ), "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)), "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 200)), - "skip_gpus": config.getoption("--skip-gpus") + "skip_gpus": config.getoption("--skip-gpus"), }, } From 16a05bb42c97d7f87e2ed0173d892ed1c739aaf0 Mon Sep 17 00:00:00 2001 From: Miguel Varela Ramos Date: Fri, 12 Feb 2021 12:46:21 +0100 Subject: [PATCH 5/5] Update tests README.md --- test/e2e/README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index cd09ffbcbc..be8638f256 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -32,9 +32,9 @@ Using a new cluster, created for testing only and deleted afterwards: pytest test/e2e/tests -k aws --aws-config ``` -**Note:** For the BatchAPI tests, the `--s3-path` option should be provided with an -AWS S3 bucket for testing purposes. It is more convenient however to define -this bucket through an environment variable, see [configuration](#configuration). +**Note:** For the BatchAPI tests, the `--s3-path` option should be provided with an AWS S3 bucket for testing purposes. +It is more convenient however to define this bucket through an environment variable, see [configuration](#configuration) +. ### GCP @@ -52,8 +52,8 @@ pytest test/e2e/tests -k gcp --gcp-config ### All Tests -You can run all tests at once, however the provider specific options should be passed -accordingly, or the test cases will be skipped. +You can run all tests at once, however the provider specific options should be passed accordingly, or the test cases +will be skipped. e.g. @@ -61,10 +61,14 @@ e.g. pytest test/e2e/tests --aws-env --gcp-env ``` +### Skip GPU Tests + +It is possible to skip GPU tests by passing the `--skip-gpus` flag to the pytest command. + ## Configuration -It is possible to configure the behaviour of the tests by defining -environment variables or a `.env` file at the project directory. +It is possible to configure the behaviour of the tests by defining environment variables or a `.env` file at the project +directory. ```dotenv # .env file