Add test for uploading large dataset to client

.github/workflows/stress-test.yml

@@ -0,0 +1,115 @@
+name: Run stress tests manually via the GitHub Actions UI
+
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  AWS_ROLE: arn:aws:iam::724664234782:role/Striveworks-Role-github_runner_npe
+  AWS_REGION: us-east-1
+
+jobs:
+  # this job is necessary to set up docker auth for the service containers for the
+  # backend functional test
+  login-to-amazon-ecr:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.AWS_ROLE }}
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: "false"
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          mask-password: "false"
+    outputs:
+      docker_username: ${{ steps.login-ecr.outputs.docker_username_724664234782_dkr_ecr_us_east_1_amazonaws_com }}
+      docker_password: ${{ steps.login-ecr.outputs.docker_password_724664234782_dkr_ecr_us_east_1_amazonaws_com }}
+
+  integration-stress-tests:
+    env:
+      COVERAGE_FILE: .coverage.integration-stress-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.AWS_ROLE }}
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: "false"
+      - uses: aws-actions/amazon-ecr-login@v1
+        id: login-ecr
+      - name: login to ECR
+        run: aws ecr get-login-password --region ${{ env.AWS_REGION }} | docker login -u AWS --password-stdin 724664234782.dkr.ecr.us-east-1.amazonaws.com
+      - name: setup backend test env
+        run: docker compose -p velour -f docker-compose.yml -f docker-compose.cicd-override.yml --env-file ./api/.env.testing up --build -d
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: install api
+        run: pip install -e ".[test]"
+        working-directory: ./api
+      - name: install client
+        run: pip install -e ".[test]"
+        working-directory: ./client
+      - run: coverage run --source="api/velour_api,client/velour" -m pytest -v integration_tests/stress_test_data_generation.py
+      - run: coverage report
+      - name: upload coverage report as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.COVERAGE_FILE }}
+          path: ${{ env.COVERAGE_FILE }}
+      - run: make stop-env
+      - run: docker compose -p velour -f docker-compose.yml -f docker-compose.cicd-override.yml --env-file ./api/.env.testing up --build -d
+        env:
+          AUTH0_DOMAIN: ${{ vars.AUTH0_DOMAIN }}
+          AUTH0_AUDIENCE: ${{ vars.AUTH0_AUDIENCE }}
+          AUTH0_ALGORITHMS: ${{ vars.AUTH0_ALGORITHMS }}
+      - name: sleep to give backend time to spin up
+        run: sleep 15
+      - name: test auth
+        run: pytest -v integration_tests/test_client_auth.py
+        env:
+          AUTH0_DOMAIN: ${{ vars.AUTH0_DOMAIN }}
+          AUTH0_AUDIENCE: ${{ vars.AUTH0_AUDIENCE }}
+          AUTH0_CLIENT_ID: ${{ vars.AUTH0_CLIENT_ID }}
+          AUTH0_CLIENT_SECRET: ${{ vars.AUTH0_CLIENT_SECRET }}
+
+  combine-coverage-report:
+    needs: [integration-stress-tests]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - run: pip install coverage
+      - uses: actions/download-artifact@v3
+        with:
+          name: .coverage.integration-stress-tests
+      - run: coverage combine
+      - run: coverage report
+      # https://nedbatchelder.com/blog/202209/making_a_coverage_badge.html
+      - run: |
+          coverage json
+          export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
+          echo "total=$TOTAL" >> $GITHUB_ENV
+      - name: "Make badge"
+        if: github.ref == 'refs/heads/main'
+        uses: schneegans/dynamic-badges-action@v1.4.0
+        with:
+          auth: ${{ secrets.GIST_SECRET }}
+          gistID: 501428c92df8d0de6805f40fb78b1363
+          filename: velour-coverage.json
+          label: Coverage
+          message: ${{ env.total }}%
+          minColorRange: 50
+          maxColorRange: 90
+          valColorRange: ${{ env.total }}

integration_tests/stress_test_data_generation.py

@@ -0,0 +1,18 @@
+# NOTE: These tests aren't run automatically on each commit. They are intended to be manually kicked-off using the [GitHub UI](https://leonardomontini.dev/github-action-manual-trigger/)
+
+import pytest
+from test_data_generation import test_generate_segmentation_data
+
+from velour.client import Client
+
+
+@pytest.fixture
+def client():
+    return Client(host="http://localhost:8000")
+
+
+def test_large_dataset_upload(client: Client):
+    """Tests the upload of a large dataset to velour (runtime: ~20 minutes)"""
+    test_generate_segmentation_data(
+        client=client, n_images=1000, n_annotations=10, n_labels=2
+    )

integration_tests/test_data_generation.py

@@ -25,13 +25,14 @@ def client():
     return Client(host="http://localhost:8000")
 
 
-def test_generate_segmentation_data(client: Client):
+def test_generate_segmentation_data(
+    client: Client,
+    n_images: int = 10,
+    n_annotations: int = 2,
+    n_labels: int = 2,
+):
     """Check that our generated dataset correctly matches our input parameters"""
 
-    n_images = 10
-    n_annotations = 10
-    n_labels = 2
-
     dataset = generate_segmentation_data(
         client=client,
         dataset_name=dset_name,
@@ -68,7 +69,7 @@ def test_generate_segmentation_data(client: Client):
             sample_image_size == sample_mask_size
         ), f"Image is size {sample_image_size}, but mask is size {sample_mask_size}"
 
-    client.delete_dataset(dset_name, timeout=30)
+    client.delete_dataset(dset_name, timeout=300)
 
 
 def test_generate_prediction_data(client: Client):
@@ -121,4 +122,4 @@ def test_generate_prediction_data(client: Client):
     }
     assert len(eval_job.metrics) > 0
 
-    client.delete_dataset(dset_name, timeout=30)
+    client.delete_dataset(dset_name, timeout=300)