From 360c539a55a7d38005df44738ca2356ffe98ff13 Mon Sep 17 00:00:00 2001 From: Ibrahim Muhammad Date: Fri, 6 Jan 2023 19:01:09 +0000 Subject: [PATCH 1/6] Make dataset tests parallelizable --- tests/integration/test_dataset.py | 23 ++++++----------------- tests/integration/test_pagination.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 17 deletions(-) create mode 100644 tests/integration/test_pagination.py diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index e9094ae98..002cbdce3 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -7,27 +7,19 @@ def test_dataset(client, rand_gen): - before = list(client.get_datasets()) - for o in before: - assert isinstance(o, Dataset) + # confirm dataset can be created name = rand_gen(str) dataset = client.create_dataset(name=name) assert dataset.name == name assert dataset.created_by() == client.get_user() assert dataset.organization() == client.get_organization() - after = list(client.get_datasets()) - assert len(after) == len(before) + 1 - assert dataset in after - - # confirm get_one returns first dataset - get_one_dataset = client.get_datasets().get_one() - assert get_one_dataset.uid == after[0].uid - - # confirm get_many(1) returns first dataset - get_many_datasets = client.get_datasets().get_many(1) - assert get_many_datasets[0].uid == after[0].uid + retrieved_dataset = client.get_dataset(dataset.uid) + assert retrieved_dataset.name == dataset.name + assert retrieved_dataset.uid == dataset.uid + assert retrieved_dataset.created_by() == dataset.created_by() + assert retrieved_dataset.organization() == dataset.organization() dataset = client.get_dataset(dataset.uid) assert dataset.name == name @@ -48,9 +40,6 @@ def test_dataset(client, rand_gen): assert dataset.description == description dataset.delete() - final = list(client.get_datasets()) - assert dataset not in final - assert set(final) == set(before) with pytest.raises(ResourceNotFoundError): dataset = client.get_dataset(dataset.uid) diff --git a/tests/integration/test_pagination.py b/tests/integration/test_pagination.py new file mode 100644 index 000000000..8b573c4dd --- /dev/null +++ b/tests/integration/test_pagination.py @@ -0,0 +1,13 @@ +from copy import copy + + +def test_get_one_and_many_dataset_order(client): + paginator = client.get_datasets() + # confirm get_one returns first dataset + all_datasets = list(paginator) + get_one_dataset = copy(paginator).get_one() + assert get_one_dataset.uid == all_datasets[0].uid + + # confirm get_many(1) returns first dataset + get_many_datasets = copy(paginator).get_many(1) + assert get_many_datasets[0].uid == all_datasets[0].uid From d0683921e5ac5db83b7f1654ef636cf201a3e84e Mon Sep 17 00:00:00 2001 From: Ibrahim Muhammad Date: Wed, 18 Jan 2023 12:27:05 -0800 Subject: [PATCH 2/6] Further test parallelization --- pytest.ini | 2 +- requirements.txt | 3 ++- .../annotation_import/test_model.py | 27 ++++++++----------- .../annotation_import/test_model_run.py | 12 +++++---- tests/integration/test_client_errors.py | 2 +- tests/integration/test_data_row_metadata.py | 15 +++++++---- tests/integration/test_data_rows.py | 16 ++++++----- tests/integration/test_project.py | 12 ++------- tests/integration/test_user_and_org.py | 4 +-- 9 files changed, 46 insertions(+), 47 deletions(-) diff --git a/pytest.ini b/pytest.ini index 9529fcb38..b56afefdd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -addopts = -s -vv -x --reruns 5 --reruns-delay 10 --durations=20 +addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20 markers = slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/requirements.txt b/requirements.txt index 4989344a7..0dadb8833 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ typeguard imagesize pyproj pygeotile -typing-extensions \ No newline at end of file +typing-extensions +pytest-xdist \ No newline at end of file diff --git a/tests/integration/annotation_import/test_model.py b/tests/integration/annotation_import/test_model.py index 457ceae04..dcfe9ef2c 100644 --- a/tests/integration/annotation_import/test_model.py +++ b/tests/integration/annotation_import/test_model.py @@ -1,31 +1,26 @@ +import pytest + from labelbox import Model +from labelbox.exceptions import ResourceNotFoundError def test_model(client, configured_project, rand_gen): - before = list(client.get_models()) - for m in before: + # Get all + models = list(client.get_models()) + for m in models: assert isinstance(m, Model) + # Create ontology = configured_project.ontology() - data = {"name": rand_gen(str), "ontology_id": ontology.uid} model = client.create_model(data["name"], data["ontology_id"]) assert model.name == data["name"] - after = list(client.get_models()) - assert len(after) == len(before) + 1 - assert model in after - + # Get one model = client.get_model(model.uid) assert model.name == data["name"] - -def test_model_delete(client, model): - before = list(client.get_models()) - - model = before[0] + # Delete model.delete() - - after = list(client.get_models()) - - assert len(before) == len(after) + 1 + with pytest.raises(ResourceNotFoundError): + client.get_model(model.uid) diff --git a/tests/integration/annotation_import/test_model_run.py b/tests/integration/annotation_import/test_model_run.py index 5b170f4aa..0029d47d0 100644 --- a/tests/integration/annotation_import/test_model_run.py +++ b/tests/integration/annotation_import/test_model_run.py @@ -52,8 +52,9 @@ def test_model_run_delete(client, model_run): models_after = list(client.get_models()) model_after = models_after[0] after = list(model_after.model_runs()) + after_uids = {mr.uid for mr in after} - assert len(before) == len(after) + 1 + assert model_run.uid not in after_uids def test_model_run_update_config(model_run_with_training_metadata): @@ -75,10 +76,11 @@ def test_model_run_get_config(model_run_with_training_metadata): def test_model_run_data_rows_delete(client, model_run_with_model_run_data_rows): - models = list(client.get_models()) - model = models[0] - model_runs = list(model.model_runs()) - model_run = model_runs[0] + # models = list(client.get_models()) + # model = models[0] + # model_runs = list(model.model_runs()) + # model_run = model_runs[0] + model_run = model_run_with_model_run_data_rows before = list(model_run.model_run_data_rows()) annotation_data_row = before[0] diff --git a/tests/integration/test_client_errors.py b/tests/integration/test_client_errors.py index 10e84cf35..fdec2f467 100644 --- a/tests/integration/test_client_errors.py +++ b/tests/integration/test_client_errors.py @@ -45,7 +45,7 @@ def test_semantic_error(client): def test_timeout_error(client, project): with pytest.raises(labelbox.exceptions.TimeoutError) as excinfo: query_str = """query getOntology { - project (where: {id: $%s}) { + project (where: {id: %s}) { ontology { normalized } diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 26eb0447d..138eca01f 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -4,6 +4,7 @@ import uuid from labelbox import DataRow, Dataset +from labelbox.exceptions import MalformedQueryException from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DataRowMetadataKind, DeleteDataRowMetadata, \ DataRowMetadataOntology, _parse_metadata_schema @@ -17,7 +18,7 @@ TEXT_SCHEMA_ID = "cko8s9r5v0001h2dk9elqdidh" CAPTURE_DT_SCHEMA_ID = "cko8sdzv70006h2dk8jg64zvb" PRE_COMPUTED_EMBEDDINGS_ID = 'ckrzang79000008l6hb5s6za1' -CUSTOM_TEXT_SCHEMA_NAME = 'custom_text' +CUSTOM_TEXT_SCHEMA_NAME = 'datarow_metadata_custom_text' FAKE_NUMBER_FIELD = { "id": FAKE_SCHEMA_ID, @@ -30,13 +31,16 @@ @pytest.fixture def mdo(client): mdo = client.get_data_row_metadata_ontology() - for schema in mdo.custom_fields: - mdo.delete_schema(schema.name) - mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + try: + mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + except MalformedQueryException: + # Do nothing if already exists + pass mdo._raw_ontology = mdo._get_ontology() mdo._raw_ontology.append(FAKE_NUMBER_FIELD) mdo._build_ontology() yield mdo + mdo.delete_schema(CUSTOM_TEXT_SCHEMA_NAME) @pytest.fixture @@ -101,7 +105,8 @@ def test_export_empty_metadata(client, configured_project_with_label, def test_get_datarow_metadata_ontology(mdo): assert len(mdo.fields) assert len(mdo.reserved_fields) - assert len(mdo.custom_fields) == 2 + # three are created by mdo fixture but there may be more + assert len(mdo.custom_fields) >= 3 split = mdo.reserved_by_name["split"]["train"] diff --git a/tests/integration/test_data_rows.py b/tests/integration/test_data_rows.py index 9bf458663..be2186a3b 100644 --- a/tests/integration/test_data_rows.py +++ b/tests/integration/test_data_rows.py @@ -7,6 +7,7 @@ import requests from labelbox import DataRow +from labelbox.exceptions import MalformedQueryException from labelbox.schema.task import Task from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadataKind import labelbox.exceptions @@ -20,18 +21,21 @@ SPLIT_SCHEMA_ID, TEST_SPLIT_ID, EMBEDDING_SCHEMA_ID, TEXT_SCHEMA_ID, CAPTURE_DT_SCHEMA_ID ].sort() -CUSTOM_TEXT_SCHEMA_NAME = "custom_text" +CUSTOM_TEXT_SCHEMA_NAME = "data_row_custom_text" @pytest.fixture def mdo(client): mdo = client.get_data_row_metadata_ontology() - for schema in mdo.custom_fields: - mdo.delete_schema(schema.name) - mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + try: + mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + except MalformedQueryException: + # Do nothing if already exists + pass mdo._raw_ontology = mdo._get_ontology() mdo._build_ontology() yield mdo + mdo.delete_schema(CUSTOM_TEXT_SCHEMA_NAME) @pytest.fixture @@ -415,7 +419,7 @@ def test_create_data_rows_with_named_metadata_field_class( "row1", DataRow.metadata_fields: [ DataRowMetadataField(name='split', value='test'), - DataRowMetadataField(name='custom_text', value='hello') + DataRowMetadataField(name=CUSTOM_TEXT_SCHEMA_NAME, value='hello') ] } @@ -430,7 +434,7 @@ def test_create_data_rows_with_named_metadata_field_class( 'value': 'test' }, { - 'name': 'custom_text', + 'name': CUSTOM_TEXT_SCHEMA_NAME, 'value': 'hello' }, ] diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 0f3d68dbf..2e74ad79b 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -11,9 +11,6 @@ def test_project(client, rand_gen): - before = list(client.get_projects()) - for o in before: - assert isinstance(o, Project) data = { "name": rand_gen(str), @@ -24,10 +21,6 @@ def test_project(client, rand_gen): assert project.name == data["name"] assert project.description == data["description"] - after = list(client.get_projects()) - assert len(after) == len(before) + 1 - assert project in after - project = client.get_project(project.uid) assert project.name == data["name"] assert project.description == data["description"] @@ -44,9 +37,8 @@ def test_project(client, rand_gen): assert project.description == update_data["description"] project.delete() - final = list(client.get_projects()) - assert project not in final - assert set(final) == set(before) + projects = list(client.get_projects()) + assert project not in projects def test_update_project_resource_tags(client, rand_gen): diff --git a/tests/integration/test_user_and_org.py b/tests/integration/test_user_and_org.py index 52dfcdebb..9f07666de 100644 --- a/tests/integration/test_user_and_org.py +++ b/tests/integration/test_user_and_org.py @@ -19,5 +19,5 @@ def test_user_and_org_projects(project): assert project.created_by() == user assert project.organization() == org - assert set(user.projects()) == user_projects.union({project}) - assert set(org.projects()) == org_projects.union({project}) + assert project in user_projects + assert project in org_projects \ No newline at end of file From aff3da0426c2d1a25cc3858ff59b8792c6b7909e Mon Sep 17 00:00:00 2001 From: Ibrahim Muhammad Date: Wed, 18 Jan 2023 12:59:09 -0800 Subject: [PATCH 3/6] Run 10 tests in parallel --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index b56afefdd..1d8241a94 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20 +addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20 -n 10 markers = slow: marks tests as slow (deselect with '-m "not slow"') From 03263cb8849264924bcc6c36afe76fd8c76d9f78 Mon Sep 17 00:00:00 2001 From: Ibrahim Muhammad Date: Wed, 18 Jan 2023 21:53:07 -0800 Subject: [PATCH 4/6] Cleanup --- tests/integration/annotation_import/test_model_run.py | 6 +----- tests/integration/test_data_row_metadata.py | 1 - tests/integration/test_data_rows.py | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/integration/annotation_import/test_model_run.py b/tests/integration/annotation_import/test_model_run.py index 0029d47d0..d7c3f925c 100644 --- a/tests/integration/annotation_import/test_model_run.py +++ b/tests/integration/annotation_import/test_model_run.py @@ -75,11 +75,7 @@ def test_model_run_get_config(model_run_with_training_metadata): assert res["batch_size"] == new_config["batch_size"] -def test_model_run_data_rows_delete(client, model_run_with_model_run_data_rows): - # models = list(client.get_models()) - # model = models[0] - # model_runs = list(model.model_runs()) - # model_run = model_runs[0] +def test_model_run_data_rows_delete(model_run_with_model_run_data_rows): model_run = model_run_with_model_run_data_rows before = list(model_run.model_run_data_rows()) diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 138eca01f..d23b4b53a 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -40,7 +40,6 @@ def mdo(client): mdo._raw_ontology.append(FAKE_NUMBER_FIELD) mdo._build_ontology() yield mdo - mdo.delete_schema(CUSTOM_TEXT_SCHEMA_NAME) @pytest.fixture diff --git a/tests/integration/test_data_rows.py b/tests/integration/test_data_rows.py index be2186a3b..8ffff4d06 100644 --- a/tests/integration/test_data_rows.py +++ b/tests/integration/test_data_rows.py @@ -35,7 +35,6 @@ def mdo(client): mdo._raw_ontology = mdo._get_ontology() mdo._build_ontology() yield mdo - mdo.delete_schema(CUSTOM_TEXT_SCHEMA_NAME) @pytest.fixture From 71f481aacae44fa34cd326da83151dc34e2e445b Mon Sep 17 00:00:00 2001 From: Ibrahim Muhammad Date: Thu, 19 Jan 2023 09:42:18 -0800 Subject: [PATCH 5/6] Enable parallel tests only for staging --- Makefile | 2 +- pytest.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f4dc3cdb0..344030301 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-staging: build -e LABELBOX_TEST_ENVIRON="staging" \ -e DA_GCP_LABELBOX_API_KEY=${DA_GCP_LABELBOX_API_KEY} \ -e LABELBOX_TEST_API_KEY_STAGING=${LABELBOX_TEST_API_KEY_STAGING} \ - local/labelbox-python:test pytest $(PATH_TO_TEST) + local/labelbox-python:test pytest -n 10 $(PATH_TO_TEST) test-prod: build docker run -it -v ${PWD}:/usr/src -w /usr/src \ diff --git a/pytest.ini b/pytest.ini index 1d8241a94..b56afefdd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20 -n 10 +addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20 markers = slow: marks tests as slow (deselect with '-m "not slow"') From 6177b59859fd3c7def65a13f6ba84e5eea0f25a5 Mon Sep 17 00:00:00 2001 From: Ibrahim Muhammad Date: Thu, 19 Jan 2023 10:56:29 -0800 Subject: [PATCH 6/6] Revert CUSTOM_TEXT_SCHEMA_NAME values --- tests/integration/test_data_row_metadata.py | 6 +++--- tests/integration/test_data_rows.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index d23b4b53a..f1c27d86b 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -18,7 +18,7 @@ TEXT_SCHEMA_ID = "cko8s9r5v0001h2dk9elqdidh" CAPTURE_DT_SCHEMA_ID = "cko8sdzv70006h2dk8jg64zvb" PRE_COMPUTED_EMBEDDINGS_ID = 'ckrzang79000008l6hb5s6za1' -CUSTOM_TEXT_SCHEMA_NAME = 'datarow_metadata_custom_text' +CUSTOM_TEXT_SCHEMA_NAME = 'custom_text' FAKE_NUMBER_FIELD = { "id": FAKE_SCHEMA_ID, @@ -104,8 +104,8 @@ def test_export_empty_metadata(client, configured_project_with_label, def test_get_datarow_metadata_ontology(mdo): assert len(mdo.fields) assert len(mdo.reserved_fields) - # three are created by mdo fixture but there may be more - assert len(mdo.custom_fields) >= 3 + # two are created by mdo fixture but there may be more + assert len(mdo.custom_fields) >= 2 split = mdo.reserved_by_name["split"]["train"] diff --git a/tests/integration/test_data_rows.py b/tests/integration/test_data_rows.py index 8ffff4d06..ade50fa0a 100644 --- a/tests/integration/test_data_rows.py +++ b/tests/integration/test_data_rows.py @@ -21,7 +21,7 @@ SPLIT_SCHEMA_ID, TEST_SPLIT_ID, EMBEDDING_SCHEMA_ID, TEXT_SCHEMA_ID, CAPTURE_DT_SCHEMA_ID ].sort() -CUSTOM_TEXT_SCHEMA_NAME = "data_row_custom_text" +CUSTOM_TEXT_SCHEMA_NAME = "custom_text" @pytest.fixture