diff --git a/Makefile b/Makefile index f4dc3cdb0..344030301 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test-staging: build -e LABELBOX_TEST_ENVIRON="staging" \ -e DA_GCP_LABELBOX_API_KEY=${DA_GCP_LABELBOX_API_KEY} \ -e LABELBOX_TEST_API_KEY_STAGING=${LABELBOX_TEST_API_KEY_STAGING} \ - local/labelbox-python:test pytest $(PATH_TO_TEST) + local/labelbox-python:test pytest -n 10 $(PATH_TO_TEST) test-prod: build docker run -it -v ${PWD}:/usr/src -w /usr/src \ diff --git a/pytest.ini b/pytest.ini index 9529fcb38..b56afefdd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -addopts = -s -vv -x --reruns 5 --reruns-delay 10 --durations=20 +addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20 markers = slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/requirements.txt b/requirements.txt index 4989344a7..0dadb8833 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ typeguard imagesize pyproj pygeotile -typing-extensions \ No newline at end of file +typing-extensions +pytest-xdist \ No newline at end of file diff --git a/tests/integration/annotation_import/test_model.py b/tests/integration/annotation_import/test_model.py index 457ceae04..dcfe9ef2c 100644 --- a/tests/integration/annotation_import/test_model.py +++ b/tests/integration/annotation_import/test_model.py @@ -1,31 +1,26 @@ +import pytest + from labelbox import Model +from labelbox.exceptions import ResourceNotFoundError def test_model(client, configured_project, rand_gen): - before = list(client.get_models()) - for m in before: + # Get all + models = list(client.get_models()) + for m in models: assert isinstance(m, Model) + # Create ontology = configured_project.ontology() - data = {"name": rand_gen(str), "ontology_id": ontology.uid} model = client.create_model(data["name"], data["ontology_id"]) assert model.name == data["name"] - after = list(client.get_models()) - assert len(after) == len(before) + 1 - assert model in after - + # Get one model = client.get_model(model.uid) assert model.name == data["name"] - -def test_model_delete(client, model): - before = list(client.get_models()) - - model = before[0] + # Delete model.delete() - - after = list(client.get_models()) - - assert len(before) == len(after) + 1 + with pytest.raises(ResourceNotFoundError): + client.get_model(model.uid) diff --git a/tests/integration/annotation_import/test_model_run.py b/tests/integration/annotation_import/test_model_run.py index 5b170f4aa..d7c3f925c 100644 --- a/tests/integration/annotation_import/test_model_run.py +++ b/tests/integration/annotation_import/test_model_run.py @@ -52,8 +52,9 @@ def test_model_run_delete(client, model_run): models_after = list(client.get_models()) model_after = models_after[0] after = list(model_after.model_runs()) + after_uids = {mr.uid for mr in after} - assert len(before) == len(after) + 1 + assert model_run.uid not in after_uids def test_model_run_update_config(model_run_with_training_metadata): @@ -74,11 +75,8 @@ def test_model_run_get_config(model_run_with_training_metadata): assert res["batch_size"] == new_config["batch_size"] -def test_model_run_data_rows_delete(client, model_run_with_model_run_data_rows): - models = list(client.get_models()) - model = models[0] - model_runs = list(model.model_runs()) - model_run = model_runs[0] +def test_model_run_data_rows_delete(model_run_with_model_run_data_rows): + model_run = model_run_with_model_run_data_rows before = list(model_run.model_run_data_rows()) annotation_data_row = before[0] diff --git a/tests/integration/test_client_errors.py b/tests/integration/test_client_errors.py index 10e84cf35..fdec2f467 100644 --- a/tests/integration/test_client_errors.py +++ b/tests/integration/test_client_errors.py @@ -45,7 +45,7 @@ def test_semantic_error(client): def test_timeout_error(client, project): with pytest.raises(labelbox.exceptions.TimeoutError) as excinfo: query_str = """query getOntology { - project (where: {id: $%s}) { + project (where: {id: %s}) { ontology { normalized } diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 26eb0447d..f1c27d86b 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -4,6 +4,7 @@ import uuid from labelbox import DataRow, Dataset +from labelbox.exceptions import MalformedQueryException from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DataRowMetadataKind, DeleteDataRowMetadata, \ DataRowMetadataOntology, _parse_metadata_schema @@ -30,9 +31,11 @@ @pytest.fixture def mdo(client): mdo = client.get_data_row_metadata_ontology() - for schema in mdo.custom_fields: - mdo.delete_schema(schema.name) - mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + try: + mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + except MalformedQueryException: + # Do nothing if already exists + pass mdo._raw_ontology = mdo._get_ontology() mdo._raw_ontology.append(FAKE_NUMBER_FIELD) mdo._build_ontology() @@ -101,7 +104,8 @@ def test_export_empty_metadata(client, configured_project_with_label, def test_get_datarow_metadata_ontology(mdo): assert len(mdo.fields) assert len(mdo.reserved_fields) - assert len(mdo.custom_fields) == 2 + # two are created by mdo fixture but there may be more + assert len(mdo.custom_fields) >= 2 split = mdo.reserved_by_name["split"]["train"] diff --git a/tests/integration/test_data_rows.py b/tests/integration/test_data_rows.py index 9bf458663..ade50fa0a 100644 --- a/tests/integration/test_data_rows.py +++ b/tests/integration/test_data_rows.py @@ -7,6 +7,7 @@ import requests from labelbox import DataRow +from labelbox.exceptions import MalformedQueryException from labelbox.schema.task import Task from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadataKind import labelbox.exceptions @@ -26,9 +27,11 @@ @pytest.fixture def mdo(client): mdo = client.get_data_row_metadata_ontology() - for schema in mdo.custom_fields: - mdo.delete_schema(schema.name) - mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + try: + mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string) + except MalformedQueryException: + # Do nothing if already exists + pass mdo._raw_ontology = mdo._get_ontology() mdo._build_ontology() yield mdo @@ -415,7 +418,7 @@ def test_create_data_rows_with_named_metadata_field_class( "row1", DataRow.metadata_fields: [ DataRowMetadataField(name='split', value='test'), - DataRowMetadataField(name='custom_text', value='hello') + DataRowMetadataField(name=CUSTOM_TEXT_SCHEMA_NAME, value='hello') ] } @@ -430,7 +433,7 @@ def test_create_data_rows_with_named_metadata_field_class( 'value': 'test' }, { - 'name': 'custom_text', + 'name': CUSTOM_TEXT_SCHEMA_NAME, 'value': 'hello' }, ] diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index e9094ae98..002cbdce3 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -7,27 +7,19 @@ def test_dataset(client, rand_gen): - before = list(client.get_datasets()) - for o in before: - assert isinstance(o, Dataset) + # confirm dataset can be created name = rand_gen(str) dataset = client.create_dataset(name=name) assert dataset.name == name assert dataset.created_by() == client.get_user() assert dataset.organization() == client.get_organization() - after = list(client.get_datasets()) - assert len(after) == len(before) + 1 - assert dataset in after - - # confirm get_one returns first dataset - get_one_dataset = client.get_datasets().get_one() - assert get_one_dataset.uid == after[0].uid - - # confirm get_many(1) returns first dataset - get_many_datasets = client.get_datasets().get_many(1) - assert get_many_datasets[0].uid == after[0].uid + retrieved_dataset = client.get_dataset(dataset.uid) + assert retrieved_dataset.name == dataset.name + assert retrieved_dataset.uid == dataset.uid + assert retrieved_dataset.created_by() == dataset.created_by() + assert retrieved_dataset.organization() == dataset.organization() dataset = client.get_dataset(dataset.uid) assert dataset.name == name @@ -48,9 +40,6 @@ def test_dataset(client, rand_gen): assert dataset.description == description dataset.delete() - final = list(client.get_datasets()) - assert dataset not in final - assert set(final) == set(before) with pytest.raises(ResourceNotFoundError): dataset = client.get_dataset(dataset.uid) diff --git a/tests/integration/test_pagination.py b/tests/integration/test_pagination.py new file mode 100644 index 000000000..8b573c4dd --- /dev/null +++ b/tests/integration/test_pagination.py @@ -0,0 +1,13 @@ +from copy import copy + + +def test_get_one_and_many_dataset_order(client): + paginator = client.get_datasets() + # confirm get_one returns first dataset + all_datasets = list(paginator) + get_one_dataset = copy(paginator).get_one() + assert get_one_dataset.uid == all_datasets[0].uid + + # confirm get_many(1) returns first dataset + get_many_datasets = copy(paginator).get_many(1) + assert get_many_datasets[0].uid == all_datasets[0].uid diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 0f3d68dbf..2e74ad79b 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -11,9 +11,6 @@ def test_project(client, rand_gen): - before = list(client.get_projects()) - for o in before: - assert isinstance(o, Project) data = { "name": rand_gen(str), @@ -24,10 +21,6 @@ def test_project(client, rand_gen): assert project.name == data["name"] assert project.description == data["description"] - after = list(client.get_projects()) - assert len(after) == len(before) + 1 - assert project in after - project = client.get_project(project.uid) assert project.name == data["name"] assert project.description == data["description"] @@ -44,9 +37,8 @@ def test_project(client, rand_gen): assert project.description == update_data["description"] project.delete() - final = list(client.get_projects()) - assert project not in final - assert set(final) == set(before) + projects = list(client.get_projects()) + assert project not in projects def test_update_project_resource_tags(client, rand_gen): diff --git a/tests/integration/test_user_and_org.py b/tests/integration/test_user_and_org.py index 52dfcdebb..9f07666de 100644 --- a/tests/integration/test_user_and_org.py +++ b/tests/integration/test_user_and_org.py @@ -19,5 +19,5 @@ def test_user_and_org_projects(project): assert project.created_by() == user assert project.organization() == org - assert set(user.projects()) == user_projects.union({project}) - assert set(org.projects()) == org_projects.union({project}) + assert project in user_projects + assert project in org_projects \ No newline at end of file