[IO-1556] Create and push E2E test

.github/workflows/e2e.yml

@@ -0,0 +1,77 @@
+name: Nightly E2E Tests
+
+on:
+  schedule:
+    - cron: "0 8 * * mon-fri"
+
+jobs:
+  e2e:
+    name: Nightly End to End Testing
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+        poetry-version: ["1.3.1"]
+        os: [ubuntu-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Setup Poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: ${{ matrix.poetry-version }}
+      - name: Install dependencies
+        run: |
+          poetry install --no-interaction --no-root --all-extras -vvv
+          pip install wheel
+          pip install --upgrade setuptools
+          pip install --editable ".[test,ml,medical,dev,ocv]"
+          pip install pytest
+      - name: Run Tests
+        run: python -m pytest e2e_tests -W ignore::DeprecationWarning
+        env:
+          E2E_API_KEY: ${{ secrets.E2E_API_KEY }}
+          E2E_ENVIRONMENT: ${{ secrets.E2E_ENVIRONMENT }}
+          E2E_TEAM: ${{ secrets.E2E_TEAM }}
+  slack-notifier:
+    name: Slack Notifier Bot
+    needs: e2e
+    if: failure()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send Slack Notification
+        run: |
+          PAYLOAD=$(cat <<EOF
+          {
+            "channel": "#${{ vars.SLACK_CHANNEL }}",
+            "username": "${{ vars.SLACK_USERNAME }}",
+            "text": "Nightly E2E run failed <!subteam^S04T5MWDHJ6>: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}",
+            "icon_emoji": "${{ vars.SLACK_ICON }}"
+          }
+          EOF
+          )
+          curl -X POST --data-urlencode "payload=$PAYLOAD" ${{ secrets.SLACK_WEBHOOK }}
+
+  slack-notifier-temp:
+    name: Slack Notifier Bot
+    needs: e2e
+    if: success()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send Slack Notification
+        run: |
+          PAYLOAD=$(cat <<EOF
+          {
+            "channel": "#${{ vars.SLACK_CHANNEL }}",
+            "username": "${{ vars.SLACK_USERNAME }}",
+            "text": "Nightly E2E run succeeded: https://github.com/v7labs/darwin-py/actions/runs/${{ github.run_id }}\n\nSanity check on master branch scheduled runs, delete me!",
+            "icon_emoji": "${{ vars.SLACK_ICON }}"
+          }
+          EOF
+          )
+          curl -X POST --data-urlencode "payload=$PAYLOAD" ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/tests.yml

@@ -61,7 +61,7 @@ jobs:
           poetry install --no-interaction --no-root --all-extras -vvv
           pip install wheel
           pip install --upgrade setuptools
-          pip install --editable ".[test,ml,medical,dev]"
-          pip install pytest pytest-describe
+          pip install --editable ".[test,ml,medical,dev,ocv]"
+          pip install pytest 
       - name: Run Tests
         run: python -m pytest -W ignore::DeprecationWarning

darwin/cli.py

@@ -45,7 +45,7 @@ def main() -> None:
     except requests.exceptions.ConnectionError:
         f._error("Darwin seems unreachable, please try again in a minute or contact support.")
     except GracefulExit as e:
-        f.error(e.message)
+        f._error(e.message)
     except Exception:  # Catch unhandled exceptions
         console = Console()
         console.print("An unexpected error occurred, please contact support, and send them the file.")

e2e_tests/conftest.py

@@ -12,8 +12,6 @@
 from e2e_tests.objects import ConfigValues, E2EDataset
 from e2e_tests.setup_tests import setup_tests, teardown_tests
 
-datasets: List[E2EDataset] = []
-
 
 def pytest_configure(config: pytest.Config) -> None:
     config.addinivalue_line("addopts", "--ignore=../tests/, ../future --capture=tee-sys")
@@ -45,8 +43,13 @@ def pytest_sessionstart(session: pytest.Session) -> None:
     session.config.cache.set("api_key", api_key)
     session.config.cache.set("team_slug", team_slug)
 
-    global datasets
     datasets = setup_tests(ConfigValues(server=server, api_key=api_key, team_slug=team_slug))
+    # pytest.datasets = datasets
+    setattr(pytest, "datasets", datasets)
+    # Set the environment variables for running CLI arguments
+    environ["DARWIN_BASE_URL"] = server
+    environ["DARWIN_TEAM"] = team_slug
+    environ["DARWIN_API_KEY"] = api_key
 
     print("Sleeping for 10 seconds to allow the server to catch up")
     sleep(10)
@@ -56,8 +59,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
     if not isinstance(session.config.cache, pytest.Cache):
         raise TypeError("Pytest caching is not enabled, but E2E tests require it")
 
-    global datasets
-
+    datasets = pytest.datasets
     if datasets is None:
         raise ValueError("Datasets were not created, so could not tear them down")
 
@@ -68,7 +70,12 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
     if server is None or api_key is None or team is None:
         raise ValueError("E2E environment variables were not cached")
 
+    del environ["DARWIN_BASE_URL"]
+    del environ["DARWIN_TEAM"]
+    del environ["DARWIN_API_KEY"]
+
     config = ConfigValues(server=server, api_key=api_key, team_slug=team)
+    assert isinstance(datasets, List)
     teardown_tests(config, datasets)
 
 

e2e_tests/helpers.py

@@ -1,7 +1,15 @@
+import re
+import tempfile
+import uuid
+from pathlib import Path
 from subprocess import run
-from typing import Optional, Tuple
+from typing import Generator, Optional, Tuple
+
+import pytest
 
 from darwin.exceptions import DarwinException
+from e2e_tests.objects import E2EDataset
+from e2e_tests.setup_tests import create_random_image
 
 
 def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tuple[int, str, str]:
@@ -38,5 +46,7 @@ def run_cli_command(command: str, working_directory: Optional[str] = None) -> Tu
             capture_output=True,
             shell=True,
         )
-
-    return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8")
+    try:
+        return result.returncode, result.stdout.decode("utf-8"), result.stderr.decode("utf-8")
+    except:
+        return result.returncode, result.stdout.decode("cp437"), result.stderr.decode("cp437")

e2e_tests/objects.py

@@ -40,12 +40,14 @@ class E2EDataset:
     name: str
     slug: str
     items: List[E2EItem]
-
-    def __init__(self, id: int, name: str, slug: Optional[str]) -> None:
+    directory: Optional[str] = None
+
+    def __init__(self, id: int, name: str, slug: Optional[str], directory: Optional[str]=None) -> None:
         self.id = id
         self.name = name
         self.slug = slug or name.lower().replace(" ", "_")
         self.items = []
+        self.directory = directory
 
     def add_item(self, item: E2EItem) -> None:
         self.items.append(item)

e2e_tests/test_darwin.py

@@ -0,0 +1,68 @@
+import re
+import tempfile
+import uuid
+from pathlib import Path
+from typing import Generator
+
+import pytest
+
+from e2e_tests.helpers import run_cli_command
+from e2e_tests.objects import E2EDataset
+from e2e_tests.setup_tests import create_random_image
+
+
+@pytest.fixture
+def new_dataset() -> E2EDataset:
+    """Create a new dataset via darwin cli and return the dataset object, complete with teardown"""
+    uuid_str = str(uuid.uuid4())
+    new_dataset_name = "test_dataset_" + uuid_str
+    result = run_cli_command(f"darwin dataset create {new_dataset_name}")
+    assert result[0] == 0
+    id_raw = re.findall(r"datasets[/\\+](\d+)", result[1])
+    assert id_raw is not None and len(id_raw) == 1
+    id = int(id_raw[0])
+    teardown_dataset = E2EDataset(id, new_dataset_name, None)
+
+    # Add the teardown dataset to the pytest object to ensure it gets deleted when pytest is done
+    pytest.datasets.append(teardown_dataset)  # type: ignore
+    return teardown_dataset
+
+
+@pytest.fixture
+def local_dataset(new_dataset: E2EDataset) -> Generator[E2EDataset, None, None]:
+    with tempfile.TemporaryDirectory() as temp_directory:
+        new_dataset.directory = temp_directory
+        yield new_dataset
+
+
+@pytest.fixture
+def local_dataset_with_images(local_dataset: E2EDataset) -> E2EDataset:
+    assert local_dataset.directory is not None
+    [create_random_image(local_dataset.slug, Path(local_dataset.directory)) for x in range(3)]
+    return local_dataset
+
+
+def test_darwin_create(local_dataset: E2EDataset) -> None:
+    """
+    Test creating a dataset via the darwin cli, heavy lifting performed
+    by the fixture which already creates a dataset and adds it to the pytest object via cli
+    """
+    assert local_dataset.id is not None
+    assert local_dataset.name is not None
+
+
+def test_darwin_push(local_dataset_with_images: E2EDataset) -> None:
+    """
+    Test pushing a dataset via the darwin cli, dataset created via fixture with images added to object
+    """
+    assert local_dataset_with_images.id is not None
+    assert local_dataset_with_images.name is not None
+    assert local_dataset_with_images.directory is not None
+    result = run_cli_command(
+        f"darwin dataset push {local_dataset_with_images.name} {local_dataset_with_images.directory}"
+    )
+    assert result[0] == 0
+
+
+if __name__ == "__main__":
+    pytest.main(["-vv", "-s", __file__])