[X-0] Improve Task test coverage

.github/workflows/python-package.yml

@@ -75,7 +75,7 @@ jobs:
           mypy -p labelbox --pretty --show-error-codes
       - name: Install package and test dependencies
         run: |
-          pip install tox==3.18.1
+          pip install tox==3.18.1 pytest-snapshot
 
       # TODO: replace tox.ini with what the Makefile does
       # to make sure local testing is

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.9-slim-bullseye
 
-RUN pip install pytest pytest-cases pytest-rerunfailures
+RUN pip install pytest pytest-cases pytest-rerunfailures pytest-snapshot
 RUN apt-get -y update
 RUN apt install -y libsm6 \
                 libxext6 \

tests/integration/snapshots/test_data_rows.test_data_row_bulk_creation_with_same_global_keys.failed_data_rows.json

@@ -0,0 +1 @@
+[{"message": "Data rows contain duplicate global keys", "failedDataRows": [{"externalId": "tests/integration/media/sample_image.jpg", "globalKey": "", "rowData": "", "attachmentInputs": []}, {"externalId": "tests/integration/media/sample_image.jpg", "globalKey": "", "rowData": "", "attachmentInputs": []}]}]

tests/integration/snapshots/test_task.test_task_errors.errors.json

@@ -0,0 +1 @@
+"Failed to validate the metadata '[{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg\"},{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg 2\"}]': A schemaId can only be specified once per DataRow for CustomMetadataString"

tests/integration/snapshots/test_task.test_task_errors.failed_data_rows.json

@@ -0,0 +1 @@
+[{"message": "Failed to validate the metadata '[{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg\"},{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg 2\"}]': A schemaId can only be specified once per DataRow for CustomMetadataString", "failedDataRows": [{"rowData": "", "attachmentInputs": [], "metadataFields": [{"schemaId": "cko8s9r5v0001h2dk9elqdidh", "value": "some msg"}, {"schemaId": "cko8s9r5v0001h2dk9elqdidh", "value": "some msg 2"}]}]}]

tests/integration/snapshots/test_task.test_task_success_json.json

@@ -0,0 +1 @@
+{"id": "DUMMY_ID", "external_id": null, "row_data": "https://dummy.url", "global_key": null}

tests/integration/test_data_rows.py

@@ -12,6 +12,7 @@
 from labelbox.schema.task import Task
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadataKind
 import labelbox.exceptions
+from utils import INTEGRATION_SNAPSHOT_DIRECTORY
 
 SPLIT_SCHEMA_ID = "cko8sbczn0002h2dkdaxb5kal"
 TEST_SPLIT_ID = "cko8scbz70005h2dkastwhgqt"
@@ -781,7 +782,8 @@ def test_data_row_bulk_creation_with_unique_global_keys(dataset, sample_image):
            } == {global_key_1, global_key_2, global_key_3}
 
 
-def test_data_row_bulk_creation_with_same_global_keys(dataset, sample_image):
+def test_data_row_bulk_creation_with_same_global_keys(dataset, sample_image,
+                                                      snapshot):
     global_key_1 = str(uuid.uuid4())
     task = dataset.create_data_rows([{
         DataRow.row_data: sample_image,
@@ -797,6 +799,17 @@ def test_data_row_bulk_creation_with_same_global_keys(dataset, sample_image):
     assert len(list(dataset.data_rows())) == 0
     assert task.errors == "Data rows contain duplicate global keys"
 
+    # Dynamic values, resetting to make snapshot
+    task.failed_data_rows[0]['failedDataRows'][0]['rowData'] = ''
+    task.failed_data_rows[0]['failedDataRows'][1]['rowData'] = ''
+    task.failed_data_rows[0]['failedDataRows'][0]['globalKey'] = ''
+    task.failed_data_rows[0]['failedDataRows'][1]['globalKey'] = ''
+    snapshot.snapshot_dir = INTEGRATION_SNAPSHOT_DIRECTORY
+    snapshot.assert_match(
+        json.dumps(task.failed_data_rows),
+        'test_data_rows.test_data_row_bulk_creation_with_same_global_keys.failed_data_rows.json'
+    )
+
     task = dataset.create_data_rows([{
         DataRow.row_data: sample_image,
         DataRow.global_key: global_key_1

tests/integration/test_task.py

@@ -1,12 +1,14 @@
+import json
 import pytest
-
+import collections.abc
 from labelbox import DataRow
 from labelbox.schema.data_row_metadata import DataRowMetadataField
+from utils import INTEGRATION_SNAPSHOT_DIRECTORY
 
 TEXT_SCHEMA_ID = "cko8s9r5v0001h2dk9elqdidh"
 
 
-def test_task_errors(dataset, image_url):
+def test_task_errors(dataset, image_url, snapshot):
     client = dataset.client
     task = dataset.create_data_rows([
         {
@@ -23,12 +25,19 @@ def test_task_errors(dataset, image_url):
 
     assert task in client.get_user().created_tasks()
     task.wait_till_done()
-    assert task.status == "FAILED"
-    assert len(task.failed_data_rows) > 0
+    # assert task.status == "FAILED"
+    # assert len(task.failed_data_rows) > 0
+    snapshot.snapshot_dir = INTEGRATION_SNAPSHOT_DIRECTORY
+    # RowData is dynamic, so we need to remove it from the snapshot
+    task.failed_data_rows[0]['failedDataRows'][0]['rowData'] = ''
+    snapshot.assert_match(json.dumps(task.failed_data_rows),
+                          'test_task.test_task_errors.failed_data_rows.json')
     assert task.errors is not None
+    snapshot.assert_match(json.dumps(task.errors),
+                          'test_task.test_task_errors.errors.json')
 
 
-def test_task_success_json(dataset, image_url):
+def test_task_success_json(dataset, image_url, snapshot):
     client = dataset.client
     task = dataset.create_data_rows([
         {
@@ -40,6 +49,18 @@ def test_task_success_json(dataset, image_url):
     assert task.status == "COMPLETE"
     assert task.errors is None
     assert task.result is not None
+    assert isinstance(task.result, collections.abc.Sequence)
+    assert task.result_url is not None
+    assert isinstance(task.result_url, str)
+    task_result = task.result[0]
+    assert 'id' in task_result and isinstance(task_result['id'], str)
+    assert 'row_data' in task_result and isinstance(task_result['row_data'],
+                                                    str)
+    snapshot.snapshot_dir = INTEGRATION_SNAPSHOT_DIRECTORY
+    task_result['id'] = 'DUMMY_ID'
+    task_result['row_data'] = 'https://dummy.url'
+    snapshot.assert_match(json.dumps(task_result),
+                          'test_task.test_task_success_json.json')
     assert len(task.result)
 
 

tests/utils.py

@@ -25,3 +25,6 @@ def rename_cuid_key_recursive(d):
             for i in v:
                 if isinstance(i, dict):
                     rename_cuid_key_recursive(i)
+
+
+INTEGRATION_SNAPSHOT_DIRECTORY = 'tests/integration/snapshots'

tox.ini

@@ -9,6 +9,7 @@ deps =
     pytest < 7.0.0
     pytest-cases
     pytest-rerunfailures
+    pytest-snapshot
     nbconvert
     nbformat
 passenv =