diff --git a/labelbox/client.py b/labelbox/client.py index c28f3fb78..ee42312b4 100644 --- a/labelbox/client.py +++ b/labelbox/client.py @@ -31,6 +31,8 @@ from labelbox.schema.model_run import ModelRun from labelbox.schema.ontology import Ontology, Tool, Classification, FeatureSchema from labelbox.schema.organization import Organization +from labelbox.schema.quality_mode import QualityMode, BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS, \ + BENCHMARK_AUTO_AUDIT_PERCENTAGE, CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS, CONSENSUS_AUTO_AUDIT_PERCENTAGE from labelbox.schema.user import User from labelbox.schema.project import Project from labelbox.schema.role import Role @@ -636,16 +638,33 @@ def create_project(self, **kwargs) -> Project: description (str): A short summary for the project media_type (MediaType): The type of assets that this project will accept queue_mode (Optional[QueueMode]): The queue mode to use - auto_audit_percentage (Optional[float]): The percentage of data rows that will require more than 1 label - auto_audit_number_of_labels (Optional[float]): Number of labels required for data rows selected for multiple labeling (auto_audit_percentage) + quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to + Benchmark Returns: A new Project object. Raises: InvalidAttributeError: If the Project type does not contain any of the attribute names given in kwargs. """ - media_type = kwargs.get("media_type") + + auto_audit_percentage = kwargs.get("auto_audit_percentage") + auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels") + if auto_audit_percentage is not None or auto_audit_number_of_labels is not None: + raise ValueError( + "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels." + ) + queue_mode = kwargs.get("queue_mode") + if queue_mode is QueueMode.Dataset: + raise ValueError( + "Dataset queue mode is deprecated. Please prefer Batch queue mode." + ) + elif queue_mode is QueueMode.Batch: + logger.warning( + "Passing a queue mode of batch is redundant and will soon no longer be supported." + ) + + media_type = kwargs.get("media_type") if media_type: if MediaType.is_supported(media_type): media_type = media_type.value @@ -658,20 +677,25 @@ def create_project(self, **kwargs) -> Project: "Creating a project without specifying media_type" " through this method will soon no longer be supported.") - if not queue_mode: - logger.warning( - "Default createProject behavior will soon be adjusted to prefer " - "batch projects. Pass in `queue_mode` parameter explicitly to opt-out for the " - "time being.") - elif queue_mode == QueueMode.Dataset: - logger.warning( - "QueueMode.Dataset will eventually be deprecated, and is no longer " - "recommended for new projects. Prefer QueueMode.Batch instead.") + quality_mode = kwargs.get("quality_mode") + if not quality_mode: + logger.info("Defaulting quality mode to Benchmark.") + + data = kwargs + data.pop("quality_mode", None) + if quality_mode is None or quality_mode is QualityMode.Benchmark: + data[ + "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS + data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE + else: + data[ + "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS + data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE return self._create(Entity.Project, { - **kwargs, + **data, **({ - 'media_type': media_type + "media_type": media_type } if media_type else {}) }) diff --git a/labelbox/orm/query.py b/labelbox/orm/query.py index 62f3846f0..4bcc292f9 100644 --- a/labelbox/orm/query.py +++ b/labelbox/orm/query.py @@ -314,7 +314,7 @@ def relationship(source, relationship, where, order_by): def create(entity, data): - """ Generats a query and parameters for creating a new DB object. + """ Generates a query and parameters for creating a new DB object. Args: entity (type): An Entity subtype indicating which kind of diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index bd912aa0b..b86579baf 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -120,12 +120,10 @@ def update(self, **kwargs): for a project is inferred through the following attributes: Benchmark: - auto_audit_number_of_labels = 1 - auto_audit_percentage = 1.0 + auto_audit_number_of_labels = 1 and auto_audit_percentage = 1.0 Consensus: - auto_audit_number_of_labels > 1 - auto_audit_percentage <= 1.0 + auto_audit_number_of_labels > 1 or auto_audit_percentage <= 1.0 Attempting to switch between benchmark and consensus modes is an invalid operation and will result in an error. diff --git a/labelbox/schema/quality_mode.py b/labelbox/schema/quality_mode.py new file mode 100644 index 000000000..abf5013ba --- /dev/null +++ b/labelbox/schema/quality_mode.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class QualityMode(str, Enum): + Benchmark = "BENCHMARK" + Consensus = "CONSENSUS" + + +BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS = 1 +BENCHMARK_AUTO_AUDIT_PERCENTAGE = 1 +CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS = 3 +CONSENSUS_AUTO_AUDIT_PERCENTAGE = 0 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 781fe6edb..ba6f97b5a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -22,6 +22,7 @@ from labelbox.schema.enums import AnnotationImportState from labelbox.schema.invite import Invite from labelbox.schema.project import Project +from labelbox.schema.quality_mode import QualityMode from labelbox.schema.queue_mode import QueueMode from labelbox.schema.user import User @@ -231,7 +232,7 @@ def project(client, rand_gen): @pytest.fixture def consensus_project(client, rand_gen): project = client.create_project(name=rand_gen(str), - auto_audit_percentage=0, + quality_mode=QualityMode.Consensus, queue_mode=QueueMode.Batch, media_type=MediaType.Image) yield project diff --git a/tests/integration/test_legacy_project.py b/tests/integration/test_legacy_project.py index b79c9ea06..85463af50 100644 --- a/tests/integration/test_legacy_project.py +++ b/tests/integration/test_legacy_project.py @@ -1,15 +1,13 @@ import pytest -from labelbox.exceptions import InvalidQueryError, MalformedQueryException -from labelbox.schema.media_type import MediaType from labelbox.schema.queue_mode import QueueMode def test_project_dataset(client, rand_gen): with pytest.raises( - MalformedQueryException, + ValueError, match= - "DataSet queue mode is deprecated. Please prefer Batch queue mode." + "Dataset queue mode is deprecated. Please prefer Batch queue mode." ): client.create_project( name=rand_gen(str), @@ -18,6 +16,21 @@ def test_project_dataset(client, rand_gen): def test_legacy_project_dataset_relationships(project, dataset): - assert [ds for ds in project.datasets()] == [] assert [p for p in dataset.projects()] == [] + + +def test_project_auto_audit_parameters(client, rand_gen): + with pytest.raises( + ValueError, + match= + "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels." + ): + client.create_project(name=rand_gen(str), auto_audit_percentage=0.5) + + with pytest.raises( + ValueError, + match= + "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels." + ): + client.create_project(name=rand_gen(str), auto_audit_number_of_labels=2) diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 94c98ee50..3566c8a1c 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -8,11 +8,11 @@ from labelbox import Project, LabelingFrontend, Dataset from labelbox.exceptions import InvalidQueryError from labelbox.schema.media_type import MediaType +from labelbox.schema.quality_mode import QualityMode from labelbox.schema.queue_mode import QueueMode def test_project(client, rand_gen): - data = { "name": rand_gen(str), "description": rand_gen(str), @@ -260,3 +260,19 @@ def test_media_type(client, project: Project, rand_gen): media_type=MediaType[media_type]) assert project.media_type == MediaType[media_type] project.delete() + + +def test_queue_mode(client, rand_gen): + project = client.create_project(name=rand_gen(str)) # defaults to benchmark + assert project.auto_audit_number_of_labels == 1 + assert project.auto_audit_percentage == 1 + + project = client.create_project(name=rand_gen(str), + quality_mode=QualityMode.Benchmark) + assert project.auto_audit_number_of_labels == 1 + assert project.auto_audit_percentage == 1 + + project = client.create_project(name=rand_gen(str), + quality_mode=QualityMode.Consensus) + assert project.auto_audit_number_of_labels == 3 + assert project.auto_audit_percentage == 0