From 3d8b59677418b163a2f5ac55c4f6a894f53205ca Mon Sep 17 00:00:00 2001 From: Kevin Kim Date: Thu, 22 Dec 2022 15:02:41 -0800 Subject: [PATCH 1/3] Fix tests that were flaky due to label creation --- tests/integration/conftest.py | 49 +++++++++++++++++-- .../test_data_row_media_attributes.py | 8 +-- tests/integration/test_data_row_metadata.py | 8 +-- tests/integration/test_export.py | 7 +-- 4 files changed, 58 insertions(+), 14 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fd2900061..c8553d908 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -119,6 +119,49 @@ def get_invites(client): return invites +def wait_for_data_row_processing(client, data_row): + """ + Do not use. Only for testing. + + Returns DataRow after waiting for it to finish processing media_attributes. + Some tests, specifically ones that rely on label export, rely on + DataRow be fully processed with media_attributes + """ + data_row_id = data_row.uid + timeout_seconds = 60 + while True: + data_row = client.get_data_row(data_row_id) + if data_row.media_attributes: + return data_row + timeout_seconds -= 2 + if timeout_seconds <= 0: + raise TimeoutError( + f"Timed out waiting for DataRow '{data_row_id}' to finish processing media_attributes" + ) + time.sleep(2) + + +def wait_for_label_processing(project): + """ + Do not use. Only for testing. + + Returns Label after waiting for it to finish processing. + If `project.labels()` is called before label is fully processed, + it may return an empty set + """ + timeout_seconds = 10 + while True: + label = project.labels().get_one() + if label is not None: + return label + timeout_seconds -= 2 + if timeout_seconds <= 0: + raise TimeoutError( + f"Timed out waiting for label for project '{project.uid}' to finish processing" + ) + time.sleep(2) + + @pytest.fixture def queries(): return SimpleNamespace(cancel_invite=cancel_invite, @@ -207,7 +250,7 @@ def datarow(dataset, image_url): }, ]) task.wait_till_done() - dr = next(dataset.data_rows()) + dr = dataset.data_rows().get_one() yield dr dr.delete() @@ -348,8 +391,8 @@ def create_label(): project.create_label = create_label project.create_label() - label = project.labels().get_one() - assert label is not None, "Cannot fetch created label" + label = wait_for_label_processing(project) + yield [project, dataset, datarow, label] for label in project.labels(): diff --git a/tests/integration/test_data_row_media_attributes.py b/tests/integration/test_data_row_media_attributes.py index 3b78744c4..692cd735f 100644 --- a/tests/integration/test_data_row_media_attributes.py +++ b/tests/integration/test_data_row_media_attributes.py @@ -1,10 +1,10 @@ from time import sleep +from conftest import wait_for_data_row_processing -def test_export_empty_media_attributes(configured_project_with_label): - project, _, _, _ = configured_project_with_label - # Wait for exporter to retrieve latest labels - sleep(10) +def test_export_empty_media_attributes(client, configured_project_with_label): + project, _, data_row, _ = configured_project_with_label + data_row = wait_for_data_row_processing(client, data_row) labels = list(project.label_generator()) assert len( labels diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 232bc718f..94e0ec06e 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -4,6 +4,7 @@ import pytest import uuid +from conftest import wait_for_data_row_processing from labelbox import DataRow, Dataset from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DataRowMetadataKind, DeleteDataRowMetadata, \ DataRowMetadataOntology, _parse_metadata_schema @@ -90,10 +91,9 @@ def make_named_metadata(dr_id) -> DataRowMetadata: return metadata -def test_export_empty_metadata(configured_project_with_label): - project, _, _, _ = configured_project_with_label - # Wait for exporter to retrieve latest labels - sleep(10) +def test_export_empty_metadata(client, configured_project_with_label): + project, _, data_row, _ = configured_project_with_label + data_row = wait_for_data_row_processing(client, data_row) labels = project.label_generator() label = next(labels) assert label.data.metadata == [] diff --git a/tests/integration/test_export.py b/tests/integration/test_export.py index d7f0689e1..85f02210b 100644 --- a/tests/integration/test_export.py +++ b/tests/integration/test_export.py @@ -1,6 +1,7 @@ from time import sleep import uuid +from conftest import wait_for_data_row_processing from labelbox.data.annotation_types.annotation import ObjectAnnotation from labelbox.schema.annotation_import import LabelImport @@ -8,6 +9,8 @@ def test_export_annotations_nested_checklist( client, configured_project_with_complex_ontology): project, data_row = configured_project_with_complex_ontology + data_row = wait_for_data_row_processing(client, data_row) + ontology = project.ontology().normalized tool = ontology["tools"][0] @@ -47,9 +50,7 @@ def test_export_annotations_nested_checklist( task = LabelImport.create_from_objects(client, project.uid, f'label-import-{uuid.uuid4()}', data) task.wait_until_done() - # Wait for exporter to retrieve latest labels - sleep(10) - labels = project.label_generator().as_list() + labels = project.label_generator() object_annotation = [ annot for annot in next(labels).annotations if isinstance(annot, ObjectAnnotation) From 16eb7665408b888437ec9d84cb0330b21f824833 Mon Sep 17 00:00:00 2001 From: Kevin Kim Date: Thu, 22 Dec 2022 15:26:54 -0800 Subject: [PATCH 2/3] Use fixtures instead --- tests/integration/conftest.py | 100 ++++++++++-------- .../test_data_row_media_attributes.py | 7 +- tests/integration/test_data_row_metadata.py | 5 +- tests/integration/test_export.py | 5 +- tests/integration/test_project_setup.py | 1 - 5 files changed, 62 insertions(+), 56 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index c8553d908..ecb47969b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -119,49 +119,6 @@ def get_invites(client): return invites -def wait_for_data_row_processing(client, data_row): - """ - Do not use. Only for testing. - - Returns DataRow after waiting for it to finish processing media_attributes. - Some tests, specifically ones that rely on label export, rely on - DataRow be fully processed with media_attributes - """ - data_row_id = data_row.uid - timeout_seconds = 60 - while True: - data_row = client.get_data_row(data_row_id) - if data_row.media_attributes: - return data_row - timeout_seconds -= 2 - if timeout_seconds <= 0: - raise TimeoutError( - f"Timed out waiting for DataRow '{data_row_id}' to finish processing media_attributes" - ) - time.sleep(2) - - -def wait_for_label_processing(project): - """ - Do not use. Only for testing. - - Returns Label after waiting for it to finish processing. - If `project.labels()` is called before label is fully processed, - it may return an empty set - """ - timeout_seconds = 10 - while True: - label = project.labels().get_one() - if label is not None: - return label - timeout_seconds -= 2 - if timeout_seconds <= 0: - raise TimeoutError( - f"Timed out waiting for label for project '{project.uid}' to finish processing" - ) - time.sleep(2) - - @pytest.fixture def queries(): return SimpleNamespace(cancel_invite=cancel_invite, @@ -347,7 +304,7 @@ def configured_project(project, client, rand_gen, image_url): @pytest.fixture def configured_project_with_label(client, rand_gen, image_url, project, dataset, - datarow): + datarow, wait_for_label_processing): """Project with a connected dataset, having one datarow Project contains an ontology with 1 bbox tool Additionally includes a create_label method for any needed extra labels @@ -450,3 +407,58 @@ def configured_project_with_complex_ontology(client, rand_gen, image_url): yield [project, data_row] dataset.delete() project.delete() + + +@pytest.fixture +def wait_for_data_row_processing(): + """ + Do not use. Only for testing. + + Returns DataRow after waiting for it to finish processing media_attributes. + Some tests, specifically ones that rely on label export, rely on + DataRow be fully processed with media_attributes + """ + + def func(client, data_row): + data_row_id = data_row.uid + timeout_seconds = 60 + while True: + data_row = client.get_data_row(data_row_id) + print(f"dr {data_row}") + if data_row.media_attributes: + return data_row + timeout_seconds -= 2 + if timeout_seconds <= 0: + raise TimeoutError( + f"Timed out waiting for DataRow '{data_row_id}' to finish processing media_attributes" + ) + time.sleep(2) + + return func + + +@pytest.fixture +def wait_for_label_processing(): + """ + Do not use. Only for testing. + + Returns Label after waiting for it to finish processing. + If `project.labels()` is called before label is fully processed, + it may return an empty set + """ + + def func(project): + timeout_seconds = 10 + while True: + label = project.labels().get_one() + print(f"LABEL: {label}") + if label is not None: + return label + timeout_seconds -= 2 + if timeout_seconds <= 0: + raise TimeoutError( + f"Timed out waiting for label for project '{project.uid}' to finish processing" + ) + time.sleep(2) + + return func diff --git a/tests/integration/test_data_row_media_attributes.py b/tests/integration/test_data_row_media_attributes.py index 692cd735f..4e75513d4 100644 --- a/tests/integration/test_data_row_media_attributes.py +++ b/tests/integration/test_data_row_media_attributes.py @@ -1,8 +1,5 @@ -from time import sleep -from conftest import wait_for_data_row_processing - - -def test_export_empty_media_attributes(client, configured_project_with_label): +def test_export_empty_media_attributes(client, configured_project_with_label, + wait_for_data_row_processing): project, _, data_row, _ = configured_project_with_label data_row = wait_for_data_row_processing(client, data_row) labels = list(project.label_generator()) diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 94e0ec06e..26eb0447d 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -1,10 +1,8 @@ from datetime import datetime -from time import sleep import pytest import uuid -from conftest import wait_for_data_row_processing from labelbox import DataRow, Dataset from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DataRowMetadataKind, DeleteDataRowMetadata, \ DataRowMetadataOntology, _parse_metadata_schema @@ -91,7 +89,8 @@ def make_named_metadata(dr_id) -> DataRowMetadata: return metadata -def test_export_empty_metadata(client, configured_project_with_label): +def test_export_empty_metadata(client, configured_project_with_label, + wait_for_data_row_processing): project, _, data_row, _ = configured_project_with_label data_row = wait_for_data_row_processing(client, data_row) labels = project.label_generator() diff --git a/tests/integration/test_export.py b/tests/integration/test_export.py index 85f02210b..9e35436a2 100644 --- a/tests/integration/test_export.py +++ b/tests/integration/test_export.py @@ -1,13 +1,12 @@ -from time import sleep import uuid -from conftest import wait_for_data_row_processing from labelbox.data.annotation_types.annotation import ObjectAnnotation from labelbox.schema.annotation_import import LabelImport def test_export_annotations_nested_checklist( - client, configured_project_with_complex_ontology): + client, configured_project_with_complex_ontology, + wait_for_data_row_processing): project, data_row = configured_project_with_complex_ontology data_row = wait_for_data_row_processing(client, data_row) diff --git a/tests/integration/test_project_setup.py b/tests/integration/test_project_setup.py index d55a1731c..8d81ba043 100644 --- a/tests/integration/test_project_setup.py +++ b/tests/integration/test_project_setup.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta, timezone import json import time -import time import pytest From 1ac0730861c3a27736031387017c348915cc7e00 Mon Sep 17 00:00:00 2001 From: Kevin Kim Date: Thu, 22 Dec 2022 15:36:46 -0800 Subject: [PATCH 3/3] wait_for_label_processing to wait for all project's labels --- tests/integration/conftest.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ecb47969b..497a894dc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -348,7 +348,7 @@ def create_label(): project.create_label = create_label project.create_label() - label = wait_for_label_processing(project) + label = wait_for_label_processing(project)[0] yield [project, dataset, datarow, label] @@ -424,7 +424,6 @@ def func(client, data_row): timeout_seconds = 60 while True: data_row = client.get_data_row(data_row_id) - print(f"dr {data_row}") if data_row.media_attributes: return data_row timeout_seconds -= 2 @@ -442,7 +441,7 @@ def wait_for_label_processing(): """ Do not use. Only for testing. - Returns Label after waiting for it to finish processing. + Returns project's labels as a list after waiting for them to finish processing. If `project.labels()` is called before label is fully processed, it may return an empty set """ @@ -450,10 +449,9 @@ def wait_for_label_processing(): def func(project): timeout_seconds = 10 while True: - label = project.labels().get_one() - print(f"LABEL: {label}") - if label is not None: - return label + labels = list(project.labels()) + if len(labels) > 0: + return labels timeout_seconds -= 2 if timeout_seconds <= 0: raise TimeoutError(