From cacc32fcf5c2a786187b20def1951bfb1ba485f6 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Wed, 13 Sep 2023 22:28:58 +0100 Subject: [PATCH 01/22] e2e testing create --- darwin/cli.py | 2 +- e2e_tests/conftest.py | 9 +++- e2e_tests/helpers.py | 16 +++++++ e2e_tests/test_create.py | 27 +++++++++++ poetry.lock | 99 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 e2e_tests/test_create.py diff --git a/darwin/cli.py b/darwin/cli.py index 7dbc372f2..3f3f5221b 100644 --- a/darwin/cli.py +++ b/darwin/cli.py @@ -45,7 +45,7 @@ def main() -> None: except requests.exceptions.ConnectionError: f._error("Darwin seems unreachable, please try again in a minute or contact support.") except GracefulExit as e: - f.error(e.message) + f._error(e.message) except Exception: # Catch unhandled exceptions console = Console() console.print("An unexpected error occurred, please contact support, and send them the file.") diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 201db7b94..8fe8639c2 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,3 +1,4 @@ +import os from os import environ from os.path import dirname, join from pathlib import Path @@ -29,6 +30,8 @@ def pytest_sessionstart(session: pytest.Session) -> None: api_key = environ.get("E2E_API_KEY") team_slug = environ.get("E2E_TEAM") + + if server is None: raise E2EEnvironmentVariableNotSet("E2E_ENVIRONMENT") @@ -47,7 +50,11 @@ def pytest_sessionstart(session: pytest.Session) -> None: global datasets datasets = setup_tests(ConfigValues(server=server, api_key=api_key, team_slug=team_slug)) - + + os.environ["DARWIN_BASE_URL"] = server + os.environ["DARWIN_TEAM"] = team_slug + os.environ["DARWIN_API_KEY"] = api_key + print("Sleeping for 10 seconds to allow the server to catch up") sleep(10) diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index f3959a322..ecb628d02 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -2,6 +2,7 @@ from typing import Optional, Tuple from darwin.exceptions import DarwinException +from e2e_tests.objects import E2EDataset def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tuple[int, str, str]: @@ -40,3 +41,18 @@ def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tu ) return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8") + + +def create_local_dataset(working_directory: str, dataset_name: str) -> Tuple[int, str, str]: + """ + Create a local dataset. + + Parameters + ---------- + working_directory : str + The working directory to create the dataset in. + dataset_name : str + The name of the dataset to create. + """ + + return run_cli_command(f"darwin dataset create {dataset_name}", working_directory) diff --git a/e2e_tests/test_create.py b/e2e_tests/test_create.py new file mode 100644 index 000000000..2648405cf --- /dev/null +++ b/e2e_tests/test_create.py @@ -0,0 +1,27 @@ +import re +import uuid + +import pytest + +from e2e_tests.helpers import run_cli_command +from e2e_tests.objects import E2EDataset + + +def test_darwin_create_and_delete() -> None: + uuid_str = str(uuid.uuid4()) + new_dataset_name = "test_dataset_" + uuid_str + result = run_cli_command(f"darwin dataset create {new_dataset_name}") + assert result[0] == 0 + assert "has been created" in result[1] + id_raw = re.search(r"/datasets/(\d+)", result[1]) + assert id_raw is not None + id = int(id_raw.group(1)) + + + result = run_cli_command(f"yes Y | darwin dataset remove {new_dataset_name}") + assert result[0] == 0 + +if __name__ == "__main__": + pytest.main(["-vv", "-s", __file__]) + + \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 52548d732..35061ac41 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "argcomplete" version = "2.1.2" description = "Bash tab completion for argparse" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -22,6 +23,7 @@ test = ["coverage", "flake8", "mypy", "pexpect", "wheel"] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -43,6 +45,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "black" version = "22.12.0" description = "The uncompromising code formatter." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -79,6 +82,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -90,6 +94,7 @@ files = [ name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -174,6 +179,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -189,6 +195,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -200,6 +207,7 @@ files = [ name = "connected-components-3d" version = "3.12.2" description = "Connected components on 2D and 3D images. Supports multiple labels." +category = "main" optional = true python-versions = ">=3.7,<4.0" files = [ @@ -247,6 +255,7 @@ numpy = "*" name = "debugpy" version = "1.6.7.post1" description = "An implementation of the Debug Adapter Protocol for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -274,6 +283,7 @@ files = [ name = "deprecation" version = "2.1.0" description = "A library to handle automated deprecations" +category = "main" optional = false python-versions = "*" files = [ @@ -284,10 +294,22 @@ files = [ [package.dependencies] packaging = "*" +[[package]] +name = "dotenv" +version = "0.0.5" +description = "Handle .env files" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "dotenv-0.0.5.tar.gz", hash = "sha256:b58d2ab3f83dbd4f8a362b21158a606bee87317a9444485566b3c8f0af847091"}, +] + [[package]] name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -302,6 +324,7 @@ test = ["pytest (>=6)"] name = "flake8" version = "6.1.0" description = "the modular source code checker: pep8 pyflakes and co" +category = "main" optional = true python-versions = ">=3.8.1" files = [ @@ -318,6 +341,7 @@ pyflakes = ">=3.1.0,<3.2.0" name = "flake8-pyproject" version = "1.2.3" description = "Flake8 plug-in loading the configuration from pyproject.toml" +category = "main" optional = true python-versions = ">= 3.6" files = [ @@ -335,6 +359,7 @@ dev = ["pyTest", "pyTest-cov"] name = "humanize" version = "4.6.0" description = "Python humanize utilities" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -352,6 +377,7 @@ tests = ["freezegun", "pytest", "pytest-cov"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -363,6 +389,7 @@ files = [ name = "importlib-metadata" version = "5.2.0" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -383,6 +410,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -401,6 +429,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -412,6 +441,7 @@ files = [ name = "isort" version = "5.11.5" description = "A Python utility / library to sort Python imports." +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -429,6 +459,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "joblib" version = "1.3.2" description = "Lightweight pipelining with Python functions" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -440,6 +471,7 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -463,6 +495,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -488,6 +521,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -499,6 +533,7 @@ files = [ name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -510,6 +545,7 @@ files = [ name = "mpire" version = "2.8.0" description = "A Python package for easy multiprocessing, but faster than multiprocessing" +category = "main" optional = false python-versions = "*" files = [ @@ -532,6 +568,7 @@ testing = ["dataclasses", "multiprocess", "multiprocess (>=0.70.15)", "numpy", " name = "mypy" version = "0.991" description = "Optional static typing for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -583,6 +620,7 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -594,6 +632,7 @@ files = [ name = "nibabel" version = "5.1.0" description = "Access a multitude of neuroimaging data formats" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -624,6 +663,7 @@ zstd = ["pyzstd (>=0.14.3)"] name = "numpy" version = "1.21.6" description = "NumPy is the fundamental package for array computing with Python." +category = "main" optional = false python-versions = ">=3.7,<3.11" files = [ @@ -664,6 +704,7 @@ files = [ name = "nvidia-cublas-cu11" version = "11.10.3.66" description = "CUBLAS native runtime libraries" +category = "main" optional = true python-versions = ">=3" files = [ @@ -679,6 +720,7 @@ wheel = "*" name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" description = "NVRTC native runtime libraries" +category = "main" optional = true python-versions = ">=3" files = [ @@ -695,6 +737,7 @@ wheel = "*" name = "nvidia-cuda-runtime-cu11" version = "11.7.99" description = "CUDA Runtime native Libraries" +category = "main" optional = true python-versions = ">=3" files = [ @@ -710,6 +753,7 @@ wheel = "*" name = "nvidia-cudnn-cu11" version = "8.5.0.96" description = "cuDNN runtime libraries" +category = "main" optional = true python-versions = ">=3" files = [ @@ -725,6 +769,7 @@ wheel = "*" name = "opencv-python-headless" version = "4.8.0.76" description = "Wrapper package for OpenCV python bindings." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -751,6 +796,7 @@ numpy = [ name = "orjson" version = "3.9.5" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -820,6 +866,7 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -831,6 +878,7 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -842,6 +890,7 @@ files = [ name = "pillow" version = "9.5.0" description = "Python Imaging Library (Fork)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -921,6 +970,7 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -932,6 +982,7 @@ files = [ name = "platformdirs" version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -950,6 +1001,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -968,6 +1020,7 @@ testing = ["pytest", "pytest-benchmark"] name = "pycodestyle" version = "2.11.0" description = "Python style guide checker" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -979,6 +1032,7 @@ files = [ name = "pydantic" version = "1.10.12" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1031,6 +1085,7 @@ email = ["email-validator (>=1.0.3)"] name = "pyflakes" version = "3.1.0" description = "passive checker of Python programs" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1042,6 +1097,7 @@ files = [ name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1056,6 +1112,7 @@ plugins = ["importlib-metadata"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1092,6 +1149,7 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1115,6 +1173,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1129,6 +1188,7 @@ cli = ["click (>=5.0)"] name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "main" optional = false python-versions = "*" files = [ @@ -1152,6 +1212,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1160,6 +1221,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1167,8 +1229,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1185,6 +1254,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1192,6 +1262,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1201,6 +1272,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1222,6 +1294,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "responses" version = "0.22.0" description = "A utility library for mocking out the `requests` Python library." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1243,6 +1316,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rich" version = "13.5.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -1262,6 +1336,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "scikit-learn" version = "1.3.0" description = "A set of python modules for machine learning and data mining" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1304,6 +1379,7 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc ( name = "scipy" version = "1.10.1" description = "Fundamental algorithms for scientific computing in Python" +category = "main" optional = true python-versions = "<3.12,>=3.8" files = [ @@ -1342,6 +1418,7 @@ test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeo name = "setuptools" version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1358,6 +1435,7 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "threadpoolctl" version = "3.2.0" description = "threadpoolctl" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1369,6 +1447,7 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1380,6 +1459,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1391,6 +1471,7 @@ files = [ name = "torch" version = "1.13.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -1431,6 +1512,7 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "torchvision" version = "0.14.1" description = "image and video datasets and models for torch deep learning" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1457,7 +1539,7 @@ files = [ [package.dependencies] numpy = "*" -pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" +pillow = ">=5.3.0,<8.3.0 || >=8.4.0" requests = "*" torch = "1.13.1" typing-extensions = "*" @@ -1469,6 +1551,7 @@ scipy = ["scipy"] name = "tqdm" version = "4.66.1" description = "Fast, Extensible Progress Meter" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1489,6 +1572,7 @@ telegram = ["requests"] name = "typed-ast" version = "1.5.5" description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1539,6 +1623,7 @@ files = [ name = "types-pyyaml" version = "6.0.12.11" description = "Typing stubs for PyYAML" +category = "main" optional = false python-versions = "*" files = [ @@ -1550,6 +1635,7 @@ files = [ name = "types-requests" version = "2.31.0.2" description = "Typing stubs for requests" +category = "main" optional = false python-versions = "*" files = [ @@ -1564,6 +1650,7 @@ types-urllib3 = "*" name = "types-toml" version = "0.10.8.7" description = "Typing stubs for toml" +category = "main" optional = true python-versions = "*" files = [ @@ -1575,6 +1662,7 @@ files = [ name = "types-urllib3" version = "1.26.25.14" description = "Typing stubs for urllib3" +category = "main" optional = false python-versions = "*" files = [ @@ -1586,6 +1674,7 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1597,6 +1686,7 @@ files = [ name = "upolygon" version = "0.1.10" description = "Collection of fast polygon operations for DL" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1618,6 +1708,7 @@ numpy = "*" name = "urllib3" version = "2.0.4" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1635,6 +1726,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "wheel" version = "0.41.1" description = "A built-package format for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1649,6 +1741,7 @@ test = ["pytest (>=6.0.0)", "setuptools (>=65)"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1670,4 +1763,4 @@ test = ["flake8-pyproject", "pytest", "responses"] [metadata] lock-version = "2.0" python-versions = ">=3.7.0,<3.11" -content-hash = "bf2caf6db46010fe08b8aac8c522e07985b2a0758c89e830a0f92bdc455c86e3" +content-hash = "eb70125650055ef52568dfa68365b63c82e357177829e3338d0fc720bfd7c720" From 87b5bd35fc4eebb3d4b62aeb89d38dea56506953 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Thu, 14 Sep 2023 18:08:51 +0100 Subject: [PATCH 02/22] test teardown refactor --- e2e_tests/conftest.py | 12 +++--- e2e_tests/helpers.py | 24 ++++-------- e2e_tests/objects.py | 6 ++- e2e_tests/test_create.py | 27 -------------- e2e_tests/test_darwin.py | 79 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 52 deletions(-) delete mode 100644 e2e_tests/test_create.py create mode 100644 e2e_tests/test_darwin.py diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 8fe8639c2..7ed303e94 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -13,8 +13,6 @@ from e2e_tests.objects import ConfigValues, E2EDataset from e2e_tests.setup_tests import setup_tests, teardown_tests -datasets: List[E2EDataset] = [] - def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line("addopts", "--ignore=../tests/, ../future --capture=tee-sys") @@ -48,9 +46,10 @@ def pytest_sessionstart(session: pytest.Session) -> None: session.config.cache.set("api_key", api_key) session.config.cache.set("team_slug", team_slug) - global datasets datasets = setup_tests(ConfigValues(server=server, api_key=api_key, team_slug=team_slug)) - + # pytest.datasets = datasets + setattr(pytest, "datasets", datasets) + # Set the environment variables for running CLI arguments os.environ["DARWIN_BASE_URL"] = server os.environ["DARWIN_TEAM"] = team_slug os.environ["DARWIN_API_KEY"] = api_key @@ -58,13 +57,11 @@ def pytest_sessionstart(session: pytest.Session) -> None: print("Sleeping for 10 seconds to allow the server to catch up") sleep(10) - def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if not isinstance(session.config.cache, pytest.Cache): raise TypeError("Pytest caching is not enabled, but E2E tests require it") - global datasets - + datasets = pytest.datasets if datasets is None: raise ValueError("Datasets were not created, so could not tear them down") @@ -76,6 +73,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: raise ValueError("E2E environment variables were not cached") config = ConfigValues(server=server, api_key=api_key, team_slug=team) + assert isinstance(datasets, List) teardown_tests(config, datasets) diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index ecb628d02..45e986734 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -1,8 +1,15 @@ +import re +import tempfile +import uuid +from pathlib import Path from subprocess import run -from typing import Optional, Tuple +from typing import Generator, Optional, Tuple + +import pytest from darwin.exceptions import DarwinException from e2e_tests.objects import E2EDataset +from e2e_tests.setup_tests import create_random_image def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tuple[int, str, str]: @@ -41,18 +48,3 @@ def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tu ) return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8") - - -def create_local_dataset(working_directory: str, dataset_name: str) -> Tuple[int, str, str]: - """ - Create a local dataset. - - Parameters - ---------- - working_directory : str - The working directory to create the dataset in. - dataset_name : str - The name of the dataset to create. - """ - - return run_cli_command(f"darwin dataset create {dataset_name}", working_directory) diff --git a/e2e_tests/objects.py b/e2e_tests/objects.py index f47dcb258..31dc8452d 100644 --- a/e2e_tests/objects.py +++ b/e2e_tests/objects.py @@ -40,12 +40,14 @@ class E2EDataset: name: str slug: str items: List[E2EItem] - - def __init__(self, id: int, name: str, slug: Optional[str]) -> None: + directory: Optional[str] = None + + def __init__(self, id: int, name: str, slug: Optional[str], directory: Optional[str]=None) -> None: self.id = id self.name = name self.slug = slug or name.lower().replace(" ", "_") self.items = [] + self.directory = directory def add_item(self, item: E2EItem) -> None: self.items.append(item) diff --git a/e2e_tests/test_create.py b/e2e_tests/test_create.py deleted file mode 100644 index 2648405cf..000000000 --- a/e2e_tests/test_create.py +++ /dev/null @@ -1,27 +0,0 @@ -import re -import uuid - -import pytest - -from e2e_tests.helpers import run_cli_command -from e2e_tests.objects import E2EDataset - - -def test_darwin_create_and_delete() -> None: - uuid_str = str(uuid.uuid4()) - new_dataset_name = "test_dataset_" + uuid_str - result = run_cli_command(f"darwin dataset create {new_dataset_name}") - assert result[0] == 0 - assert "has been created" in result[1] - id_raw = re.search(r"/datasets/(\d+)", result[1]) - assert id_raw is not None - id = int(id_raw.group(1)) - - - result = run_cli_command(f"yes Y | darwin dataset remove {new_dataset_name}") - assert result[0] == 0 - -if __name__ == "__main__": - pytest.main(["-vv", "-s", __file__]) - - \ No newline at end of file diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py new file mode 100644 index 000000000..6c0a2ef8a --- /dev/null +++ b/e2e_tests/test_darwin.py @@ -0,0 +1,79 @@ +import re +import tempfile +import uuid +from pathlib import Path +from typing import Generator + +import pytest + +from e2e_tests.helpers import run_cli_command +from e2e_tests.objects import E2EDataset +from e2e_tests.setup_tests import create_random_image + + +@pytest.fixture +def new_dataset() -> E2EDataset: + uuid_str = str(uuid.uuid4()) + new_dataset_name = "test_dataset_" + uuid_str + result = run_cli_command(f"darwin dataset create {new_dataset_name}") + assert result[0] == 0 + id_raw = re.search(r"/datasets/(\d+)", result[1]) + assert id_raw is not None + id = int(id_raw.group(1)) + teardown_dataset = E2EDataset(id, new_dataset_name, None) + pytest.datasets.append(teardown_dataset) # type: ignore + return teardown_dataset + +@pytest.fixture +def local_dataset(new_dataset: E2EDataset) -> Generator[E2EDataset, None, None]: + with tempfile.TemporaryDirectory() as temp_directory: + new_dataset.directory = temp_directory + yield new_dataset + +@pytest.fixture +def local_dataset_with_images(local_dataset: E2EDataset) -> Generator[E2EDataset, None, None]: + assert local_dataset.directory is not None + [create_random_image(local_dataset.slug, Path(local_dataset.directory)) for x in range(3)] + yield local_dataset + + +def test_darwin_create(local_dataset: E2EDataset) -> None: + assert local_dataset.id is not None + assert local_dataset.name is not None + + + + +def test_darwin_push(local_dataset_with_images: E2EDataset) -> None: + assert local_dataset_with_images.id is not None + assert local_dataset_with_images.name is not None + assert local_dataset_with_images.directory is not None + result = run_cli_command(f"darwin dataset push {local_dataset_with_images.name} {local_dataset_with_images.directory}") + assert result[0] == 0 + + + + + +# def test_darwin_import() -> None: +# uuid_str = str(uuid.uuid4()) +# new_dataset_name = "test_dataset_" + uuid_str +# result = run_cli_command(f"darwin dataset create {new_dataset_name}") +# assert result[0] == 0 +# assert "has been created" in result[1] +# id_raw = re.search(r"/datasets/(\d+)", result[1]) +# assert id_raw is not None +# id = int(id_raw.group(1)) + +# with tempfile.TemporaryDirectory() as temp_directory: +# images = [create_random_image(uuid_str, Path(temp_directory)) for x in range(5)] +# result = run_cli_command(f"darwin dataset push {new_dataset_name} {temp_directory}") +# assert result[0] == 0 + +# result = run_cli_command(f"yes Y | darwin dataset remove {new_dataset_name}") +# assert result[0] == 0 + +if __name__ == "__main__": + pytest.main(["-vv", "-s", __file__]) + + \ No newline at end of file From be854d666b56987b9d862bd3881dcc4bad5e2611 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Fri, 15 Sep 2023 09:35:30 +0100 Subject: [PATCH 03/22] cleanup --- e2e_tests/test_darwin.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py index 6c0a2ef8a..f9f5ad266 100644 --- a/e2e_tests/test_darwin.py +++ b/e2e_tests/test_darwin.py @@ -13,6 +13,7 @@ @pytest.fixture def new_dataset() -> E2EDataset: + """Create a new dataset via darwin cli and return the dataset object, complete with teardown""" uuid_str = str(uuid.uuid4()) new_dataset_name = "test_dataset_" + uuid_str result = run_cli_command(f"darwin dataset create {new_dataset_name}") @@ -21,6 +22,8 @@ def new_dataset() -> E2EDataset: assert id_raw is not None id = int(id_raw.group(1)) teardown_dataset = E2EDataset(id, new_dataset_name, None) + + # Add the teardown dataset to the pytest object to ensure it gets deleted when pytest is done pytest.datasets.append(teardown_dataset) # type: ignore return teardown_dataset @@ -31,13 +34,17 @@ def local_dataset(new_dataset: E2EDataset) -> Generator[E2EDataset, None, None]: yield new_dataset @pytest.fixture -def local_dataset_with_images(local_dataset: E2EDataset) -> Generator[E2EDataset, None, None]: +def local_dataset_with_images(local_dataset: E2EDataset) -> E2EDataset: assert local_dataset.directory is not None [create_random_image(local_dataset.slug, Path(local_dataset.directory)) for x in range(3)] - yield local_dataset + return local_dataset def test_darwin_create(local_dataset: E2EDataset) -> None: + """ + Test creating a dataset via the darwin cli, heavy lifting performed + by the fixture which already creates a dataset and adds it to the pytest object via cli + """ assert local_dataset.id is not None assert local_dataset.name is not None @@ -45,33 +52,15 @@ def test_darwin_create(local_dataset: E2EDataset) -> None: def test_darwin_push(local_dataset_with_images: E2EDataset) -> None: + """ + Test pushing a dataset via the darwin cli, dataset created via fixture with images added to object + """ assert local_dataset_with_images.id is not None assert local_dataset_with_images.name is not None assert local_dataset_with_images.directory is not None result = run_cli_command(f"darwin dataset push {local_dataset_with_images.name} {local_dataset_with_images.directory}") assert result[0] == 0 - - - - -# def test_darwin_import() -> None: -# uuid_str = str(uuid.uuid4()) -# new_dataset_name = "test_dataset_" + uuid_str -# result = run_cli_command(f"darwin dataset create {new_dataset_name}") -# assert result[0] == 0 -# assert "has been created" in result[1] -# id_raw = re.search(r"/datasets/(\d+)", result[1]) -# assert id_raw is not None -# id = int(id_raw.group(1)) - -# with tempfile.TemporaryDirectory() as temp_directory: -# images = [create_random_image(uuid_str, Path(temp_directory)) for x in range(5)] -# result = run_cli_command(f"darwin dataset push {new_dataset_name} {temp_directory}") -# assert result[0] == 0 - -# result = run_cli_command(f"yes Y | darwin dataset remove {new_dataset_name}") -# assert result[0] == 0 if __name__ == "__main__": pytest.main(["-vv", "-s", __file__]) From a4ade3ff725199643e4971dbbb9df2dddc4c70cd Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Fri, 15 Sep 2023 17:08:08 +0100 Subject: [PATCH 04/22] e2e environment variables --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 039d005e2..b7cd38941 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,4 +97,5 @@ jobs: E2E_API_KEY: ${{ secrets.E2E_API_KEY }} E2E_ENVIRONMENT: ${{ secrets.E2E_ENVIRONMENT }} + \ No newline at end of file From 96de81390b7e2094e105e9dbca839cc0bc39b2fb Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Fri, 15 Sep 2023 17:20:35 +0100 Subject: [PATCH 05/22] e2e environment variables --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b7cd38941..8ed2d6b3f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,6 +96,7 @@ jobs: env: E2E_API_KEY: ${{ secrets.E2E_API_KEY }} E2E_ENVIRONMENT: ${{ secrets.E2E_ENVIRONMENT }} + E2E_TEAM: ${{ secrets.E2E_TEAM }} \ No newline at end of file From ce1ae1417e1af6d1a28f3d3f10b8e8e3cc15846b Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Mon, 18 Sep 2023 09:18:24 +0100 Subject: [PATCH 06/22] testing fix for windows --- e2e_tests/test_darwin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py index f9f5ad266..77ca17483 100644 --- a/e2e_tests/test_darwin.py +++ b/e2e_tests/test_darwin.py @@ -18,9 +18,9 @@ def new_dataset() -> E2EDataset: new_dataset_name = "test_dataset_" + uuid_str result = run_cli_command(f"darwin dataset create {new_dataset_name}") assert result[0] == 0 - id_raw = re.search(r"/datasets/(\d+)", result[1]) - assert id_raw is not None - id = int(id_raw.group(1)) + id_raw = re.findall(r"/datasets/(\d+)", result[1]) + assert id_raw is not None and len(id_raw) == 1 + id = int(id_raw[0]) teardown_dataset = E2EDataset(id, new_dataset_name, None) # Add the teardown dataset to the pytest object to ensure it gets deleted when pytest is done From 7a703ce609498c1df2ea4d2b9279aaa263bbe781 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Mon, 18 Sep 2023 09:37:27 +0100 Subject: [PATCH 07/22] diagnostics --- e2e_tests/test_darwin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py index 77ca17483..2a51b15f8 100644 --- a/e2e_tests/test_darwin.py +++ b/e2e_tests/test_darwin.py @@ -19,6 +19,8 @@ def new_dataset() -> E2EDataset: result = run_cli_command(f"darwin dataset create {new_dataset_name}") assert result[0] == 0 id_raw = re.findall(r"/datasets/(\d+)", result[1]) + print(result) + print(id_raw) assert id_raw is not None and len(id_raw) == 1 id = int(id_raw[0]) teardown_dataset = E2EDataset(id, new_dataset_name, None) From f50e29d1dcb2d6d2c4464aa9b4bca84a7eee9432 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Mon, 18 Sep 2023 10:09:51 +0100 Subject: [PATCH 08/22] potential windows fix --- e2e_tests/test_darwin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py index 2a51b15f8..63efacc09 100644 --- a/e2e_tests/test_darwin.py +++ b/e2e_tests/test_darwin.py @@ -18,7 +18,7 @@ def new_dataset() -> E2EDataset: new_dataset_name = "test_dataset_" + uuid_str result = run_cli_command(f"darwin dataset create {new_dataset_name}") assert result[0] == 0 - id_raw = re.findall(r"/datasets/(\d+)", result[1]) + id_raw = re.findall(r"datasets[/|\\+](\d+)", result[1]) print(result) print(id_raw) assert id_raw is not None and len(id_raw) == 1 From b56eb84180fcb517b7f3d7cc86fd253946d9e363 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Mon, 18 Sep 2023 10:23:05 +0100 Subject: [PATCH 09/22] potential windows fix --- e2e_tests/test_darwin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py index 63efacc09..0230659b8 100644 --- a/e2e_tests/test_darwin.py +++ b/e2e_tests/test_darwin.py @@ -18,9 +18,7 @@ def new_dataset() -> E2EDataset: new_dataset_name = "test_dataset_" + uuid_str result = run_cli_command(f"darwin dataset create {new_dataset_name}") assert result[0] == 0 - id_raw = re.findall(r"datasets[/|\\+](\d+)", result[1]) - print(result) - print(id_raw) + id_raw = re.findall(r"datasets[/\\+](\d+)", result[1]) assert id_raw is not None and len(id_raw) == 1 id = int(id_raw[0]) teardown_dataset = E2EDataset(id, new_dataset_name, None) From 04945286e68abf81dba75bae8486e61ccc4ed1de Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Mon, 18 Sep 2023 10:44:41 +0100 Subject: [PATCH 10/22] potential windows fix --- e2e_tests/helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index 45e986734..f6c3092e6 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -46,5 +46,7 @@ def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tu capture_output=True, shell=True, ) - - return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8") + try: + return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8") + except: + return result.returncode, result.stdout.decode("cp437"), result.stderr.decode("cp437") \ No newline at end of file From cf09f0e71cd1d75afc1e1bcfbbe723108f45d890 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Mon, 18 Sep 2023 10:59:22 +0100 Subject: [PATCH 11/22] cleanup --- e2e_tests/conftest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 7ed303e94..2b8c90c3f 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,4 +1,3 @@ -import os from os import environ from os.path import dirname, join from pathlib import Path @@ -50,9 +49,9 @@ def pytest_sessionstart(session: pytest.Session) -> None: # pytest.datasets = datasets setattr(pytest, "datasets", datasets) # Set the environment variables for running CLI arguments - os.environ["DARWIN_BASE_URL"] = server - os.environ["DARWIN_TEAM"] = team_slug - os.environ["DARWIN_API_KEY"] = api_key + environ["DARWIN_BASE_URL"] = server + environ["DARWIN_TEAM"] = team_slug + environ["DARWIN_API_KEY"] = api_key print("Sleeping for 10 seconds to allow the server to catch up") sleep(10) From 7168e1991d45decd0332a3b85f73953b640099af Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 15:53:53 +0100 Subject: [PATCH 12/22] e2e nightly@ --- .github/workflows/e2e.yml | 57 +++++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 6 ++-- 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..2d84b9563 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,57 @@ +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * mon-fri' +jobs: + e2e: + name: Nightly End to End Testing + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + poetry-version: ["1.3.1"] + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Upgrade pip + run: python -m pip install --upgrade pip + - name: Setup Poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: ${{ matrix.poetry-version }} + - name: Install dependencies + run: | + poetry install --no-interaction --no-root --all-extras -vvv + pip install wheel + pip install --upgrade setuptools + pip install --editable ".[test,ml,medical,dev,ocv]" + pip install pytest + - name: Run Tests + run: python -m pytest e2e_tests -W ignore::DeprecationWarning + env: + E2E_API_KEY: ${{ secrets.E2E_API_KEY }} + E2E_ENVIRONMENT: ${{ secrets.E2E_ENVIRONMENT }} + E2E_TEAM: ${{ secrets.E2E_TEAM }} + slack-notifier: + name: Slack Notifier Bot + needs: e2e + if: failure() + runs-on: ubuntu-latest + steps: + - name: Send Slack Notification + run: | + PAYLOAD=$(cat < Date: Tue, 19 Sep 2023 16:13:45 +0100 Subject: [PATCH 13/22] e2e nightly test --- .github/workflows/e2e.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2d84b9563..c09171afc 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,7 +1,5 @@ on: - workflow_dispatch: - schedule: - - cron: '0 2 * * mon-fri' + pull_request: jobs: e2e: name: Nightly End to End Testing From b9448d18b102365d24df5b7d984a84a4ec72637d Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 16:19:33 +0100 Subject: [PATCH 14/22] e2e nightly test + slack --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c09171afc..9c169e7fa 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -37,7 +37,7 @@ jobs: slack-notifier: name: Slack Notifier Bot needs: e2e - if: failure() + if: always() runs-on: ubuntu-latest steps: - name: Send Slack Notification @@ -46,7 +46,7 @@ jobs: { "channel": "#${{ vars.SLACK_CHANNEL }}", "username": "${{ vars.SLACK_USERNAME }}", - "text": "Nightly E2E run failed @Darwin.py: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", + "text": "Nightly E2E run succeeded @Darwin.py: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", "icon_emoji": "${{ vars.SLACK_ICON }}" }) EOF From 097cf07b737fcb8be8911fac532f985596c67baa Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 16:42:52 +0100 Subject: [PATCH 15/22] e2e nightly test + slack --- .github/workflows/e2e.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9c169e7fa..0a1bffc41 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -48,8 +48,9 @@ jobs: "username": "${{ vars.SLACK_USERNAME }}", "text": "Nightly E2E run succeeded @Darwin.py: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", "icon_emoji": "${{ vars.SLACK_ICON }}" - }) + } EOF + ) curl -X POST --data-urlencode "payload=$PAYLOAD" ${{ secrets.SLACK_WEBHOOK }} env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} \ No newline at end of file + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} From 138b648fe4a7fccbb5809ee03ea9ef86890dbba8 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 16:55:05 +0100 Subject: [PATCH 16/22] slack fix? --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0a1bffc41..e0d65daec 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -37,7 +37,7 @@ jobs: slack-notifier: name: Slack Notifier Bot needs: e2e - if: always() + if: failure() runs-on: ubuntu-latest steps: - name: Send Slack Notification @@ -46,7 +46,7 @@ jobs: { "channel": "#${{ vars.SLACK_CHANNEL }}", "username": "${{ vars.SLACK_USERNAME }}", - "text": "Nightly E2E run succeeded @Darwin.py: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", + "text": "Nightly E2E run failed @darwin-py-devs: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", "icon_emoji": "${{ vars.SLACK_ICON }}" } EOF From 0f1d07550dd5a84f774b5d836b0e347b9c756415 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 17:10:23 +0100 Subject: [PATCH 17/22] slack fix? --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e0d65daec..515068301 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -37,7 +37,7 @@ jobs: slack-notifier: name: Slack Notifier Bot needs: e2e - if: failure() + if: always() runs-on: ubuntu-latest steps: - name: Send Slack Notification From e427cdf6df89c695b39c800ad19121a40da62aec Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 17:18:01 +0100 Subject: [PATCH 18/22] slack fix? --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 515068301..2c6f6a6e5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -46,7 +46,7 @@ jobs: { "channel": "#${{ vars.SLACK_CHANNEL }}", "username": "${{ vars.SLACK_USERNAME }}", - "text": "Nightly E2E run failed @darwin-py-devs: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", + "text": "Nightly E2E run failed : https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}", "icon_emoji": "${{ vars.SLACK_ICON }}" } EOF From 8fabf19bb02d9ddcb21732351d535dbd60003f98 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 17:28:40 +0100 Subject: [PATCH 19/22] slack fix --- .github/workflows/e2e.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2c6f6a6e5..405e753de 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,5 +1,7 @@ +name: Nightly E2E Tests on: - pull_request: + schedule: + - cron: "0 8 * * mon-fri" jobs: e2e: name: Nightly End to End Testing @@ -37,7 +39,7 @@ jobs: slack-notifier: name: Slack Notifier Bot needs: e2e - if: always() + if: failure() runs-on: ubuntu-latest steps: - name: Send Slack Notification From 324bb31d4d5391b6e3a2b9e6f0d3c98a576b47d1 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 17:41:06 +0100 Subject: [PATCH 20/22] cleanup --- .github/workflows/e2e.yml | 4 ++-- .github/workflows/tests.yml | 32 -------------------------------- e2e_tests/conftest.py | 9 ++++++--- e2e_tests/helpers.py | 2 +- e2e_tests/test_darwin.py | 16 ++++++++-------- 5 files changed, 17 insertions(+), 46 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 405e753de..36e5832a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,7 +1,9 @@ name: Nightly E2E Tests + on: schedule: - cron: "0 8 * * mon-fri" + jobs: e2e: name: Nightly End to End Testing @@ -54,5 +56,3 @@ jobs: EOF ) curl -X POST --data-urlencode "payload=$PAYLOAD" ${{ secrets.SLACK_WEBHOOK }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 266949143..d56f4db84 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,38 +65,6 @@ jobs: pip install pytest - name: Run Tests run: python -m pytest -W ignore::DeprecationWarning - e2e: - strategy: - fail-fast: false - matrix: - python-version: ["3.10"] - poetry-version: ["1.3.1"] - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Upgrade pip - run: python -m pip install --upgrade pip - - name: Setup Poetry - uses: abatilo/actions-poetry@v2 - with: - poetry-version: ${{ matrix.poetry-version }} - - name: Install dependencies - run: | - poetry install --no-interaction --no-root --all-extras -vvv - pip install wheel - pip install --upgrade setuptools - pip install --editable ".[test,ml,medical,dev]" - pip install pytest - - name: Run Tests - run: python -m pytest e2e_tests -W ignore::DeprecationWarning - env: - E2E_API_KEY: ${{ secrets.E2E_API_KEY }} - E2E_ENVIRONMENT: ${{ secrets.E2E_ENVIRONMENT }} - E2E_TEAM: ${{ secrets.E2E_TEAM }} diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 2b8c90c3f..207437e23 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -27,8 +27,6 @@ def pytest_sessionstart(session: pytest.Session) -> None: api_key = environ.get("E2E_API_KEY") team_slug = environ.get("E2E_TEAM") - - if server is None: raise E2EEnvironmentVariableNotSet("E2E_ENVIRONMENT") @@ -52,10 +50,11 @@ def pytest_sessionstart(session: pytest.Session) -> None: environ["DARWIN_BASE_URL"] = server environ["DARWIN_TEAM"] = team_slug environ["DARWIN_API_KEY"] = api_key - + print("Sleeping for 10 seconds to allow the server to catch up") sleep(10) + def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if not isinstance(session.config.cache, pytest.Cache): raise TypeError("Pytest caching is not enabled, but E2E tests require it") @@ -71,6 +70,10 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if server is None or api_key is None or team is None: raise ValueError("E2E environment variables were not cached") + del environ["DARWIN_BASE_URL"] + del environ["DARWIN_TEAM"] + del environ["DARWIN_API_KEY"] + config = ConfigValues(server=server, api_key=api_key, team_slug=team) assert isinstance(datasets, List) teardown_tests(config, datasets) diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index f6c3092e6..1ea9a34b6 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -49,4 +49,4 @@ def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tu try: return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8") except: - return result.returncode, result.stdout.decode("cp437"), result.stderr.decode("cp437") \ No newline at end of file + return result.returncode, result.stdout.decode("cp437"), result.stderr.decode("cp437") diff --git a/e2e_tests/test_darwin.py b/e2e_tests/test_darwin.py index 0230659b8..e3201d793 100644 --- a/e2e_tests/test_darwin.py +++ b/e2e_tests/test_darwin.py @@ -22,17 +22,19 @@ def new_dataset() -> E2EDataset: assert id_raw is not None and len(id_raw) == 1 id = int(id_raw[0]) teardown_dataset = E2EDataset(id, new_dataset_name, None) - + # Add the teardown dataset to the pytest object to ensure it gets deleted when pytest is done - pytest.datasets.append(teardown_dataset) # type: ignore + pytest.datasets.append(teardown_dataset) # type: ignore return teardown_dataset + @pytest.fixture def local_dataset(new_dataset: E2EDataset) -> Generator[E2EDataset, None, None]: with tempfile.TemporaryDirectory() as temp_directory: new_dataset.directory = temp_directory yield new_dataset + @pytest.fixture def local_dataset_with_images(local_dataset: E2EDataset) -> E2EDataset: assert local_dataset.directory is not None @@ -42,14 +44,12 @@ def local_dataset_with_images(local_dataset: E2EDataset) -> E2EDataset: def test_darwin_create(local_dataset: E2EDataset) -> None: """ - Test creating a dataset via the darwin cli, heavy lifting performed + Test creating a dataset via the darwin cli, heavy lifting performed by the fixture which already creates a dataset and adds it to the pytest object via cli """ assert local_dataset.id is not None assert local_dataset.name is not None - - def test_darwin_push(local_dataset_with_images: E2EDataset) -> None: """ @@ -58,11 +58,11 @@ def test_darwin_push(local_dataset_with_images: E2EDataset) -> None: assert local_dataset_with_images.id is not None assert local_dataset_with_images.name is not None assert local_dataset_with_images.directory is not None - result = run_cli_command(f"darwin dataset push {local_dataset_with_images.name} {local_dataset_with_images.directory}") + result = run_cli_command( + f"darwin dataset push {local_dataset_with_images.name} {local_dataset_with_images.directory}" + ) assert result[0] == 0 if __name__ == "__main__": pytest.main(["-vv", "-s", __file__]) - - \ No newline at end of file From a41eb2a858d22825ac4058f2649f77a3441abb0e Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 17:41:58 +0100 Subject: [PATCH 21/22] cleanup --- .github/workflows/tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d56f4db84..249d5b8cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,6 +65,3 @@ jobs: pip install pytest - name: Run Tests run: python -m pytest -W ignore::DeprecationWarning - - - From a6e0319011b68ecfb743eac8972c090d8ccfac24 Mon Sep 17 00:00:00 2001 From: Nathan Perkins Date: Tue, 19 Sep 2023 17:48:49 +0100 Subject: [PATCH 22/22] scheduled task sanity check --- .github/workflows/e2e.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 36e5832a3..81750048a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -56,3 +56,22 @@ jobs: EOF ) curl -X POST --data-urlencode "payload=$PAYLOAD" ${{ secrets.SLACK_WEBHOOK }} + + slack-notifier-temp: + name: Slack Notifier Bot + needs: e2e + if: success() + runs-on: ubuntu-latest + steps: + - name: Send Slack Notification + run: | + PAYLOAD=$(cat <