From 737769aa496c5fe83eba510c4b8a03f76311048d Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Thu, 4 Mar 2021 09:38:45 -0500 Subject: [PATCH 01/13] check metadata type --- labelbox/schema/asset_metadata.py | 10 +++++++--- labelbox/schema/data_row.py | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/labelbox/schema/asset_metadata.py b/labelbox/schema/asset_metadata.py index c5aa0ff9c..a8a108f52 100644 --- a/labelbox/schema/asset_metadata.py +++ b/labelbox/schema/asset_metadata.py @@ -9,9 +9,13 @@ class AssetMetadata(DbObject): meta_type (str): IMAGE, VIDEO, TEXT, or IMAGE_OVERLAY meta_value (str): URL to an external file or a string of text """ - VIDEO = "VIDEO" - IMAGE = "IMAGE" - TEXT = "TEXT" + + VALID_TYPES = { + "VIDEO", + "IMAGE", + "TEXT", + "IMAGE_OVERLAY" + } meta_type = Field.String("meta_type") meta_value = Field.String("meta_value") diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index df46f1902..8975521a7 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -2,6 +2,7 @@ from labelbox.orm.db_object import DbObject, Updateable, BulkDeletable from labelbox.orm.model import Entity, Field, Relationship from labelbox.pagination import PaginatedCollection +from labelbox.schema.asset_metadata import AssetMetadata class DataRow(DbObject, Updateable, BulkDeletable): @@ -34,6 +35,8 @@ class DataRow(DbObject, Updateable, BulkDeletable): metadata = Relationship.ToMany("AssetMetadata", False, "metadata") predictions = Relationship.ToMany("Prediction", False) + + @staticmethod def bulk_delete(data_rows): """ Deletes all the given DataRows. @@ -60,6 +63,9 @@ def create_metadata(self, meta_type, meta_value): Returns: `AssetMetadata` DB object. """ + if meta_type not in AssetMetadata.VALID_TYPES: + raise ValueError(f"metadata must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}") + meta_type_param = "metaType" meta_value_param = "metaValue" data_row_id_param = "dataRowId" From cef0b7a9692ebdab9352e845551da9ca7bec9601 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Fri, 5 Mar 2021 08:05:29 -0500 Subject: [PATCH 02/13] docs and validation --- labelbox/schema/asset_metadata.py | 12 +- labelbox/schema/bulk_import_request.py | 2 +- labelbox/schema/data_row.py | 4 +- labelbox/schema/dataset.py | 28 +- labelbox/schema/project.py | 80 +++-- labelbox/schema/webhook.py | 32 +- tests/integration/conftest.py | 305 ++++++++---------- tests/integration/test_asset_metadata.py | 6 + .../test_labeling_parameter_overrides.py | 23 ++ tests/integration/test_project.py | 6 + 10 files changed, 283 insertions(+), 215 deletions(-) diff --git a/labelbox/schema/asset_metadata.py b/labelbox/schema/asset_metadata.py index a8a108f52..91e2eca5f 100644 --- a/labelbox/schema/asset_metadata.py +++ b/labelbox/schema/asset_metadata.py @@ -10,12 +10,12 @@ class AssetMetadata(DbObject): meta_value (str): URL to an external file or a string of text """ - VALID_TYPES = { - "VIDEO", - "IMAGE", - "TEXT", - "IMAGE_OVERLAY" - } + VIDEO = "VIDEO" + IMAGE = "IMAGE" + TEXT= "TEXT" + IMAGE_OVERLAY = "IMAGE_OVERLAY" + + SUPPORTED_TYPES = {VIDEO, IMAGE, TEXT, IMAGE_OVERLAY} meta_type = Field.String("meta_type") meta_value = Field.String("meta_value") diff --git a/labelbox/schema/bulk_import_request.py b/labelbox/schema/bulk_import_request.py index 8b1185c03..440ca7ad7 100644 --- a/labelbox/schema/bulk_import_request.py +++ b/labelbox/schema/bulk_import_request.py @@ -493,7 +493,7 @@ class Config: extra = 'forbid' @staticmethod - def determinants(parent_cls) -> None: + def determinants(parent_cls) -> List[str]: #This is a hack for better error messages return [ k for k, v in parent_cls.__fields__.items() diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index 8975521a7..632ab4165 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -1,7 +1,6 @@ from labelbox.orm import query from labelbox.orm.db_object import DbObject, Updateable, BulkDeletable from labelbox.orm.model import Entity, Field, Relationship -from labelbox.pagination import PaginatedCollection from labelbox.schema.asset_metadata import AssetMetadata @@ -34,7 +33,6 @@ class DataRow(DbObject, Updateable, BulkDeletable): labels = Relationship.ToMany("Label", True) metadata = Relationship.ToMany("AssetMetadata", False, "metadata") predictions = Relationship.ToMany("Prediction", False) - @staticmethod @@ -64,7 +62,7 @@ def create_metadata(self, meta_type, meta_value): `AssetMetadata` DB object. """ if meta_type not in AssetMetadata.VALID_TYPES: - raise ValueError(f"metadata must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}") + raise ValueError(f"metadata type must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}") meta_type_param = "metaType" meta_value_param = "metaValue" diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index 994e840a9..7aa5d6dd1 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -1,4 +1,5 @@ import json +import logging from multiprocessing.dummy import Pool as ThreadPool import os @@ -163,7 +164,7 @@ def convert_item(item): task._user = user return task - def data_row_for_external_id(self, external_id): + def data_rows_for_external_id(self, external_id, limit = 10): """ Convenience method for getting a single `DataRow` belonging to this `Dataset` that has the given `external_id`. @@ -182,10 +183,29 @@ def data_row_for_external_id(self, external_id): where = DataRow.external_id == external_id data_rows = self.data_rows(where=where) - # Get at most two data_rows. - data_rows = [row for row, _ in zip(data_rows, range(2))] + # Get at most `limit` data_rows. + data_rows = [row for row, _ in zip(data_rows, range(limit))] - if len(data_rows) != 1: + if not len(data_rows): raise ResourceNotFoundError(DataRow, where) + return data_rows + + def data_row_for_external_id(self, external_id): + """ Convenience method for getting a single `DataRow` belonging to this + `Dataset` that has the given `external_id`. + + Args: + external_id (str): External ID of the sought `DataRow`. + Returns: + A single `DataRow` with the given ID. + + Raises: + labelbox.exceptions.ResourceNotFoundError: If there is no `DataRow` + in this `DataSet` with the given external ID, or if there are + multiple `DataRows` for it. + """ + data_rows = self.data_rows_for_external_id(external_id=external_id, limit = 2) + if len(data_rows) > 1: + logging.warn("More than one data_row has the provided external_id. Use function data_rows_for_external_id to fetch all") return data_rows[0] diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index 928f94927..a382d78be 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -1,6 +1,7 @@ from collections import namedtuple from datetime import datetime, timezone import json +from labelbox.schema.data_row import DataRow import logging from pathlib import Path import time @@ -88,6 +89,7 @@ def create_label(self, **kwargs): # deprecated and we don't want the Py client lib user to know # about them. At the same time they're connected to a Label at # label creation in a non-standard way (connect via name). + logger.warning("This function is deprecated and is not compatible with the new editor.") Label = Entity.Label @@ -249,18 +251,56 @@ def setup(self, labeling_frontend, labeling_frontend_options): timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") self.update(setup_complete=timestamp) + def validate_labeling_parameter_overrides(self, data): + for idx, row in enumerate(data): + if len(row) != 3: + raise TypeError(f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items") + data_row, priority, num_labels = row + if not isinstance(data_row, DataRow): + raise TypeError(f"Datarow should be be of type DataRow. Found {data_row}") + + for name, value in [["priority", priority], ["Number of labels", num_labels]]: + if not isinstance(value, int): + raise TypeError(f"{name} must be an int. Found {type(value)} for data_row {data_row}") + if value < 1: + raise ValueError(f"{name} must be greater than 0 for data_row {data_row}") + def set_labeling_parameter_overrides(self, data): """ Adds labeling parameter overrides to this project. + Priority: + * data will be labeled in priority order with the lower priority numbers being labeled first + - Minimum priority is 1. + * Priority is not the queue position. The position is determined by the relative priority. + - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] + will be assigned in the following order: [data_row_2, data_row_1, data_row_3] + * datarows with parameter overrides will appear before datarows without overrides + * The priority only effects items in the queue and assigning a priority will not automatically add the item back the queue + - If a datarow has already been labeled this will not have an effect until it is added back into the queue + + Number of labels: + * The number times a data row should be labeled + * This will create duplicates in a project + * The queue will never assign the same datarow to a labeler more than once + - if the number of labels is greater than the number of labelers working on a project then + the extra items will get stuck in the queue (thsi can be fixed by removing the override at any time). + * This can add items to the queue even if they have already been labeled but they will only be assigned to members who have not labeled that image before. + * Set this to 1 if you only want to effect the priority. + + + See information on priority here: + https://docs.labelbox.com/en/configure-editor/queue-system#reservation-system + >>> project.set_labeling_parameter_overrides([ >>> (data_row_1, 2, 3), (data_row_2, 1, 4)]) Args: data (iterable): An iterable of tuples. Each tuple must contain - (DataRow, priority, numberOfLabels) for the new override. + (Union[DataRow, datarow_id], priority, numberOfLabels) for the new override. Returns: bool, indicates if the operation was a success. """ + self.validate_labeling_parameter_overrides(data) data_str = ",\n".join( "{dataRow: {id: \"%s\"}, priority: %d, numLabels: %d }" % (data_row.uid, priority, num_labels) @@ -275,6 +315,8 @@ def set_labeling_parameter_overrides(self, data): def unset_labeling_parameter_overrides(self, data_rows): """ Removes labeling parameter overrides to this project. + * This will remove unlabeled duplicates in the queue. + Args: data_rows (iterable): An iterable of DataRows. Returns: @@ -290,12 +332,19 @@ def unset_labeling_parameter_overrides(self, data_rows): return res["project"]["unsetLabelingParameterOverrides"]["success"] def upsert_review_queue(self, quota_factor): - """ Reinitiates the review queue for this project. + """ Sets the the proportion of total assets in a project to review. + + More information can be found here: + https://docs.labelbox.com/en/quality-assurance/review-labels#configure-review-percentage Args: quota_factor (float): Which part (percentage) of the queue to reinitiate. Between 0 and 1. """ + + if (quota_factor > 1.) or (quota_factor < 0.): + raise ValueError("Quota factor must be in the range of [0,1]") + id_param = "projectId" quota_param = "quotaFactor" query_str = """mutation UpsertReviewQueuePyApi($%s: ID!, $%s: Float!){ @@ -307,25 +356,6 @@ def upsert_review_queue(self, quota_factor): quota_param: quota_factor }) - def extend_reservations(self, queue_type): - """ Extends all the current reservations for the current user on the given - queue type. - - Args: - queue_type (str): Either "LabelingQueue" or "ReviewQueue" - Returns: - int, the number of reservations that were extended. - """ - if queue_type not in ("LabelingQueue", "ReviewQueue"): - raise InvalidQueryError("Unsupported queue type: %s" % queue_type) - - id_param = "projectId" - query_str = """mutation ExtendReservationsPyApi($%s: ID!){ - extendReservations(projectId:$%s queueType:%s)}""" % ( - id_param, id_param, queue_type) - res = self.client.execute(query_str, {id_param: self.uid}) - return res["extendReservations"] - def create_prediction_model(self, name, version): """ Creates a PredictionModel connected to a Legacy Editor Project. @@ -335,6 +365,9 @@ def create_prediction_model(self, name, version): Returns: A newly created PredictionModel. """ + + logger.warning("This function is deprecated and is not compatible with the new editor.") + PM = Entity.PredictionModel model = self.client._create(PM, { PM.name.name: name, @@ -360,6 +393,8 @@ def create_prediction(self, label, data_row, prediction_model=None): is None and this Project's active_prediction_model is also None. """ + logger.warning("This function is deprecated and is not compatible with the new editor.") + if prediction_model is None: prediction_model = self.active_prediction_model() if prediction_model is None: @@ -433,6 +468,7 @@ def upload_annotations( Returns: BulkImportRequest """ + if isinstance(annotations, str) or isinstance(annotations, Path): def _is_url_valid(url: Union[str, Path]) -> bool: @@ -493,6 +529,8 @@ class LabelingParameterOverride(DbObject): number_of_labels = Field.Int("number_of_labels") + + LabelerPerformance = namedtuple( "LabelerPerformance", "user count seconds_per_label, total_time_labeling " "consensus average_benchmark_agreement last_activity_time") diff --git a/labelbox/schema/webhook.py b/labelbox/schema/webhook.py index 65802600c..1961cc039 100644 --- a/labelbox/schema/webhook.py +++ b/labelbox/schema/webhook.py @@ -1,7 +1,9 @@ +import logging from labelbox.orm import query from labelbox.orm.db_object import DbObject, Updateable from labelbox.orm.model import Entity, Field, Relationship +logger = logging.getLogger(__name__) class Webhook(DbObject, Updateable): """ Represents a server-side rule for sending notifications to a web-server @@ -27,6 +29,19 @@ class Webhook(DbObject, Updateable): LABEL_UPDATED = "LABEL_UPDATED" LABEL_DELETED = "LABEL_DELETED" + REVIEW_CREATED ="REVIEW_CREATED" + REVIEW_UPDATED ="REVIEW_UPDATED" + REVIEW_DELETED = "REVIEW_DELETED" + + SUPPORTED_TOPICS = { + LABEL_CREATED, + LABEL_UPDATED, + LABEL_DELETED, + REVIEW_CREATED, + REVIEW_UPDATED, + REVIEW_DELETED + } + updated_at = Field.DateTime("updated_at") created_at = Field.DateTime("created_at") url = Field.String("url") @@ -41,7 +56,7 @@ def create(client, topics, url, secret, project): client (Client): The Labelbox client used to connect to the server. topics (list of str): A list of topics this Webhook should - get notifications for. + get notifications for. Must be one of Webhook.SUPPORTED_TOPICS url (str): The URL to which notifications should be sent by the Labelbox server. secret (str): A secret key used for signing notifications. @@ -50,7 +65,12 @@ def create(client, topics, url, secret, project): events in your organization. Returns: A newly created Webhook. + + Information on configuring your server can be found here (this is where the url points to and the secret is set). + https://docs.labelbox.com/en/configure-editor/webhooks-setup#setup-steps + """ + project_str = "" if project is None \ else ("project:{id:\"%s\"}," % project.uid) @@ -65,6 +85,9 @@ def create(client, topics, url, secret, project): organization = Relationship.ToOne("Organization") project = Relationship.ToOne("Project") + def delete(self): + self.update(status = "INACTIVE") + def update(self, topics=None, url=None, status=None): """ Updates this Webhook. @@ -72,7 +95,14 @@ def update(self, topics=None, url=None, status=None): topics (list of str): The new topics value, optional. url (str): The new URL value, optional. status (str): The new status value, optional. + + If values are set to None then there are no updates made to that field. + + The following code will delete the webhook. + >>> self.update(status = Webhook.INACTIVE) + """ + # Webhook has a custom `update` function due to custom types # in `status` and `topics` fields. topics_str = "" if topics is None \ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a1f2f88d4..409d130fe 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -69,7 +69,6 @@ def client(environ: str): @pytest.fixture def rand_gen(): - def gen(field_type): if field_type is str: return "".join(ascii_letters[randint(0, @@ -125,103 +124,33 @@ def sample_video() -> str: assert os.path.exists(path_to_video) return path_to_video - @pytest.fixture def ontology(): - bbox_tool = { - 'required': - False, - 'name': - 'bbox', - 'tool': - 'rectangle', - 'color': - '#a23030', - 'classifications': [{ - 'required': False, - 'instructions': 'nested', - 'name': 'nested', - 'type': 'radio', - 'options': [{ - 'label': 'radio_option_1', - 'value': 'radio_value_1' - }] - }] - } - polygon_tool = { - 'required': False, - 'name': 'polygon', - 'tool': 'polygon', - 'color': '#FF34FF', - 'classifications': [] - } - polyline_tool = { - 'required': False, - 'name': 'polyline', - 'tool': 'line', - 'color': '#FF4A46', - 'classifications': [] - } - point_tool = { - 'required': False, - 'name': 'point--', - 'tool': 'point', - 'color': '#008941', - 'classifications': [] - } - entity_tool = { - 'required': False, - 'name': 'entity--', - 'tool': 'named-entity', - 'color': '#006FA6', - 'classifications': [] - } - segmentation_tool = { - 'required': False, - 'name': 'segmentation--', - 'tool': 'superpixel', - 'color': '#A30059', - 'classifications': [] - } - checklist = { - 'required': - False, - 'instructions': - 'checklist', - 'name': - 'checklist', - 'type': - 'checklist', - 'options': [{ - 'label': 'option1', - 'value': 'option1' - }, { - 'label': 'option2', - 'value': 'option2' - }, { - 'label': 'optionN', - 'value': 'optionn' - }] - } - free_form_text = { - 'required': False, - 'instructions': 'text', - 'name': 'text', - 'type': 'text', - 'options': [] - } + bbox_tool = {'required': False, 'name': 'bbox', 'tool': 'rectangle', 'color': '#a23030', 'classifications' : [ + { + 'required': False, 'instructions': 'nested', 'name': 'nested', 'type': 'radio', 'options': [ + { + 'label': 'radio_option_1', 'value': 'radio_value_1' + } + ] + } + ]} + polygon_tool = {'required': False, 'name': 'polygon', 'tool': 'polygon', 'color': '#FF34FF', 'classifications' : []} + polyline_tool = {'required': False, 'name': 'polyline', 'tool': 'line', 'color': '#FF4A46', 'classifications': []} + point_tool = {'required': False, 'name': 'point--', 'tool': 'point', 'color': '#008941', 'classifications': []} + entity_tool = {'required': False, 'name': 'entity--', 'tool': 'named-entity', 'color': '#006FA6', 'classifications': []} + segmentation_tool = {'required': False, 'name': 'segmentation--', 'tool': 'superpixel', 'color': '#A30059', 'classifications': []} + checklist = {'required': False, 'instructions': 'checklist', 'name': 'checklist', 'type': 'checklist', 'options': [{'label': 'option1', 'value': 'option1'}, {'label': 'option2', 'value': 'option2'}, {'label': 'optionN', 'value': 'optionn'}]} + free_form_text = {'required': False, 'instructions': 'text', 'name': 'text', 'type': 'text', 'options': []} ####Why does this break? #Adding radio buttons causes the whole ontology to be invalid.. - #radio_buttons = {'required': False, 'instructions': 'radio_tool', 'name': 'radio_tool', 'type': 'radio', 'options': [{'label': 'yes', 'value': 'yes'}, {'label': 'no', 'value': 'no'}]}, + #radio_buttons = {'required': False, 'instructions': 'radio_tool', 'name': 'radio_tool', 'type': 'radio', 'options': [{'label': 'yes', 'value': 'yes'}, {'label': 'no', 'value': 'no'}]}, #ontology = {"tools": [bbox_tool, polygon_tool, polyline_tool, point_tool, entity_tool, segmentation_tool], "classifications": [radio_buttons, checklist, free_form_text]} #### - tools = [ - bbox_tool, polygon_tool, polyline_tool, point_tool, entity_tool, - segmentation_tool - ] + tools = [bbox_tool, polygon_tool, polyline_tool, point_tool, entity_tool, segmentation_tool] classifications = [checklist, free_form_text] - return {"tools": tools, "classifications": classifications} + return {"tools" : tools, "classifications" : classifications} @pytest.fixture @@ -231,7 +160,7 @@ def configured_project(client, rand_gen, ontology): client.get_labeling_frontends( where=LabelingFrontend.name == "editor"))[0] project.setup(editor, ontology) - dataset = client.create_dataset(name=rand_gen(str)) + dataset = client.create_dataset(name = rand_gen(str)) for _ in range(len(ontology['tools']) + len(ontology['classifications'])): dataset.create_data_row(row_data=IMG_URL) project.datasets.connect(dataset) @@ -240,6 +169,7 @@ def configured_project(client, rand_gen, ontology): dataset.delete() + @pytest.fixture def prediction_id_mapping(configured_project): #Maps tool types to feature schema ids @@ -247,159 +177,176 @@ def prediction_id_mapping(configured_project): inferences = [] datarows = [d for d in list(configured_project.datasets())[0].data_rows()] result = {} - + for idx, tool in enumerate(ontology['tools'] + ontology['classifications']): if 'tool' in tool: tool_type = tool['tool'] else: tool_type = tool['type'] result[tool_type] = { - "uuid": str(uuid.uuid4()), + "uuid" : str(uuid.uuid4()), "schemaId": tool['featureSchemaId'], "dataRow": { "id": datarows[idx].uid, }, - 'tool': tool + 'tool' : tool } return result - + @pytest.fixture def polygon_inference(prediction_id_mapping): polygon = prediction_id_mapping['polygon'].copy() - polygon.update({ - "polygon": [{ - "x": 147.692, - "y": 118.154 - }, { - "x": 142.769, - "y": 404.923 - }, { - "x": 57.846, - "y": 318.769 - }, { - "x": 28.308, - "y": 169.846 - }] - }) + polygon.update( +{ + "polygon": [{ + "x": 147.692, + "y": 118.154 + }, { + "x": 142.769, + "y": 404.923 + }, { + "x": 57.846, + "y": 318.769 + }, { + "x": 28.308, + "y": 169.846 + }] + } + ) del polygon['tool'] return polygon - @pytest.fixture def rectangle_inference(prediction_id_mapping): rectangle = prediction_id_mapping['rectangle'].copy() - rectangle.update({ - "bbox": { - "top": 48, - "left": 58, - "height": 865, - "width": 1512 - }, - 'classifications': [{ - "schemaId": - rectangle['tool']['classifications'][0]['featureSchemaId'], - "answer": { - "schemaId": - rectangle['tool']['classifications'][0]['options'][0] - ['featureSchemaId'] - } - }] - }) + rectangle.update( {"bbox": { + "top": 48, + "left": 58, + "height": 865, + "width": 1512 + }, + 'classifications' : [{ + "schemaId" : rectangle['tool']['classifications'][0]['featureSchemaId'], + "answer": {"schemaId": rectangle['tool']['classifications'][0]['options'][0]['featureSchemaId'] } + }] + }) del rectangle['tool'] return rectangle - @pytest.fixture def line_inference(prediction_id_mapping): line = prediction_id_mapping['line'].copy() - line.update( - {"line": [{ - "x": 147.692, - "y": 118.154 - }, { - "x": 150.692, - "y": 160.154 - }]}) - del line['tool'] + line.update({ + "line": [{ + "x": 147.692, + "y": 118.154 + }, + { + "x": 150.692, + "y": 160.154 + }] + }) + del line['tool'] return line - @pytest.fixture def point_inference(prediction_id_mapping): point = prediction_id_mapping['point'].copy() - point.update({"point": {"x": 147.692, "y": 118.154}}) - del point['tool'] + point.update({ + "point": { + "x": 147.692, + "y": 118.154 + }} + ) + del point['tool'] return point - @pytest.fixture def entity_inference(prediction_id_mapping): entity = prediction_id_mapping['named-entity'].copy() - entity.update({"location": {"start": 67, "end": 128}}) - del entity['tool'] + entity.update({"location" : { + "start" : 67, + "end" : 128 + }}) + del entity['tool'] return entity - @pytest.fixture def segmentation_inference(prediction_id_mapping): segmentation = prediction_id_mapping['superpixel'].copy() - segmentation.update( - {'mask': { - 'instanceURI': "sampleuri", - 'colorRGB': [0, 0, 0] - }}) - del segmentation['tool'] - return segmentation - + segmentation.update({'mask' : { + 'instanceURI' : "sampleuri", + 'colorRGB' : [0,0,0] + } + }) + del segmentation['tool'] + return segmentation @pytest.fixture def checklist_inference(prediction_id_mapping): checklist = prediction_id_mapping['checklist'].copy() - checklist.update({ - 'answers': [{ - 'schemaId': checklist['tool']['options'][0]['featureSchemaId'] - }] - }) - del checklist['tool'] - return checklist - + checklist.update({'answers' : [ + { + 'schemaId' : checklist['tool']['options'][0]['featureSchemaId'] + } + ] + }) + del checklist['tool'] + return checklist @pytest.fixture def text_inference(prediction_id_mapping): text = prediction_id_mapping['text'].copy() - text.update({'answer': "free form text..."}) + text.update({ + 'answer' : "free form text..." + }) del text['tool'] - return text + return text @pytest.fixture def video_checklist_inference(prediction_id_mapping): checklist = prediction_id_mapping['checklist'].copy() - checklist.update({ - 'answers': [{ - 'schemaId': checklist['tool']['options'][0]['featureSchemaId'] - }] - }) + checklist.update({'answers' : [ + { + 'schemaId' : checklist['tool']['options'][0]['featureSchemaId'] + } + ] + }) - checklist.update( - {"frames": [{ + checklist.update({ + "frames": [ + { "start": 7, "end": 13, - }, { + }, + { "start": 18, "end": 19, - }]}) - del checklist['tool'] - return checklist + } + ]}) + del checklist['tool'] + return checklist + + @pytest.fixture -def predictions(polygon_inference, rectangle_inference, line_inference, - entity_inference, segmentation_inference, checklist_inference, - text_inference): +def predictions(polygon_inference, + rectangle_inference, + line_inference, + entity_inference, + segmentation_inference, + checklist_inference, + text_inference): return [ - polygon_inference, rectangle_inference, line_inference, - entity_inference, segmentation_inference, checklist_inference, + polygon_inference, + rectangle_inference, + line_inference, + entity_inference, + segmentation_inference, + checklist_inference, text_inference ] + \ No newline at end of file diff --git a/tests/integration/test_asset_metadata.py b/tests/integration/test_asset_metadata.py index ebc8a632b..2f3c64296 100644 --- a/tests/integration/test_asset_metadata.py +++ b/tests/integration/test_asset_metadata.py @@ -19,6 +19,9 @@ def test_asset_metadata_crud(project, dataset, rand_gen): assert asset.meta_value == "Value" assert len(list(data_row.metadata())) == 1 + with pytest.raises(ValueError): + data_row.create_metadata("NOT_SUPPORTED_TYPE", "Value") + # Check that filtering and sorting is prettily disabled with pytest.raises(InvalidQueryError) as exc_info: data_row.metadata(where=AssetMetadata.meta_value == "x") @@ -28,3 +31,6 @@ def test_asset_metadata_crud(project, dataset, rand_gen): data_row.metadata(order_by=AssetMetadata.meta_value.asc) assert exc_info.value.message == \ "Relationship DataRow.metadata doesn't support sorting" + + + diff --git a/tests/integration/test_labeling_parameter_overrides.py b/tests/integration/test_labeling_parameter_overrides.py index 30cc2f4cf..fd4c1adfd 100644 --- a/tests/integration/test_labeling_parameter_overrides.py +++ b/tests/integration/test_labeling_parameter_overrides.py @@ -1,3 +1,4 @@ +import pytest from labelbox import DataRow IMG_URL = "https://picsum.photos/200/300" @@ -33,4 +34,26 @@ def test_labeling_parameter_overrides(project, rand_gen): # currently this doesn't work so the count remains 3 assert len(list(project.labeling_parameter_overrides())) == 1 + + with pytest.raises(TypeError): + data = [(data_rows[12], "yoo", 3)] + project.set_labeling_parameter_overrides(data) + + with pytest.raises(TypeError): + data = [(data_rows[12], 3, "yoo")] + project.set_labeling_parameter_overrides(data) + + with pytest.raises(TypeError): + data = [(data_rows[12].uid, 1, 3)] + project.set_labeling_parameter_overrides(data) + + with pytest.raises(TypeError): + data = [(data_rows[12].uid, 0, 3)] + project.set_labeling_parameter_overrides(data) + + with pytest.raises(TypeError): + data = [(data_rows[12].uid, 1, 0)] + project.set_labeling_parameter_overrides(data) + dataset.delete() + diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index aadead361..d57ed4e67 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -58,6 +58,12 @@ def test_project_filtering(client, rand_gen): def test_upsert_review_queue(project): project.upsert_review_queue(0.6) + with pytest.raises(ValueError): + project.upsert_review_queue(1.001) + + with pytest.raises(ValueError): + project.upsert_review_queue(-0.001) + def test_extend_reservations(project): assert project.extend_reservations("LabelingQueue") == 0 From 9637c810886ab8e40d91d3625f8024e1251a19b5 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Sun, 7 Mar 2021 17:28:22 -0500 Subject: [PATCH 03/13] minor changes --- labelbox/schema/project.py | 74 ++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index a382d78be..f0921a8f6 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -89,7 +89,9 @@ def create_label(self, **kwargs): # deprecated and we don't want the Py client lib user to know # about them. At the same time they're connected to a Label at # label creation in a non-standard way (connect via name). - logger.warning("This function is deprecated and is not compatible with the new editor.") + logger.warning( + "This function is deprecated and is not compatible with the new editor." + ) Label = Entity.Label @@ -254,29 +256,39 @@ def setup(self, labeling_frontend, labeling_frontend_options): def validate_labeling_parameter_overrides(self, data): for idx, row in enumerate(data): if len(row) != 3: - raise TypeError(f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items") + raise TypeError( + f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items" + ) data_row, priority, num_labels = row if not isinstance(data_row, DataRow): - raise TypeError(f"Datarow should be be of type DataRow. Found {data_row}") + raise TypeError( + f"Datarow should be be of type DataRow. Found {data_row}") - for name, value in [["priority", priority], ["Number of labels", num_labels]]: + for name, value in [["priority", priority], + ["Number of labels", num_labels]]: if not isinstance(value, int): - raise TypeError(f"{name} must be an int. Found {type(value)} for data_row {data_row}") + raise TypeError( + f"{name} must be an int. Found {type(value)} for data_row {data_row}" + ) if value < 1: - raise ValueError(f"{name} must be greater than 0 for data_row {data_row}") + raise ValueError( + f"{name} must be greater than 0 for data_row {data_row}" + ) def set_labeling_parameter_overrides(self, data): """ Adds labeling parameter overrides to this project. Priority: - * data will be labeled in priority order with the lower priority numbers being labeled first + * data will be labeled in priority order + - lower numbers labeled first - Minimum priority is 1. - * Priority is not the queue position. The position is determined by the relative priority. + * Priority is not the queue position. + - The position is determined by the relative priority. - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] will be assigned in the following order: [data_row_2, data_row_1, data_row_3] * datarows with parameter overrides will appear before datarows without overrides - * The priority only effects items in the queue and assigning a priority will not automatically add the item back the queue - - If a datarow has already been labeled this will not have an effect until it is added back into the queue + * The priority only effects items in the queue + - Assigning a priority will not automatically add the item back into the queue Number of labels: * The number times a data row should be labeled @@ -284,8 +296,9 @@ def set_labeling_parameter_overrides(self, data): * The queue will never assign the same datarow to a labeler more than once - if the number of labels is greater than the number of labelers working on a project then the extra items will get stuck in the queue (thsi can be fixed by removing the override at any time). - * This can add items to the queue even if they have already been labeled but they will only be assigned to members who have not labeled that image before. - * Set this to 1 if you only want to effect the priority. + * This can add items to the queue (even if they have already been labeled) + - New copies will only be assigned to members who have not labeled that same datarow before. + * Setting this to 1 will result in the default behavior (no duplicates) See information on priority here: @@ -366,7 +379,9 @@ def create_prediction_model(self, name, version): A newly created PredictionModel. """ - logger.warning("This function is deprecated and is not compatible with the new editor.") + logger.warning( + "This function is deprecated and is not compatible with the new editor." + ) PM = Entity.PredictionModel model = self.client._create(PM, { @@ -393,7 +408,9 @@ def create_prediction(self, label, data_row, prediction_model=None): is None and this Project's active_prediction_model is also None. """ - logger.warning("This function is deprecated and is not compatible with the new editor.") + logger.warning( + "This function is deprecated and is not compatible with the new editor." + ) if prediction_model is None: prediction_model = self.active_prediction_model() @@ -449,11 +466,10 @@ def enable_model_assisted_labeling(self, toggle: bool = True) -> bool: "showingPredictionsToLabelers"] def upload_annotations( - self, - name: str, - annotations: Union[str, Union[str, Path], Iterable[dict]], - validate = True - ) -> 'BulkImportRequest': # type: ignore + self, + name: str, + annotations: Union[str, Union[str, Path], Iterable[dict]], + validate=True) -> 'BulkImportRequest': # type: ignore """ Uploads annotations to a new Editor project. Args: @@ -468,7 +484,7 @@ def upload_annotations( Returns: BulkImportRequest """ - + if isinstance(annotations, str) or isinstance(annotations, Path): def _is_url_valid(url: Union[str, Path]) -> bool: @@ -486,13 +502,11 @@ def _is_url_valid(url: Union[str, Path]) -> bool: return bool(parsed.scheme) and bool(parsed.netloc) if _is_url_valid(annotations): - return BulkImportRequest.create_from_url( - client=self.client, - project_id=self.uid, - name=name, - url=str(annotations), - validate = validate - ) + return BulkImportRequest.create_from_url(client=self.client, + project_id=self.uid, + name=name, + url=str(annotations), + validate=validate) else: path = Path(annotations) if not path.exists(): @@ -512,12 +526,12 @@ def _is_url_valid(url: Union[str, Path]) -> bool: project_id=self.uid, name=name, predictions=annotations, # type: ignore - validate = validate - ) + validate=validate) else: raise ValueError( f'Invalid annotations given of type: {type(annotations)}') + class LabelingParameterOverride(DbObject): """ Customizes the order of assets in the label queue. @@ -529,8 +543,6 @@ class LabelingParameterOverride(DbObject): number_of_labels = Field.Int("number_of_labels") - - LabelerPerformance = namedtuple( "LabelerPerformance", "user count seconds_per_label, total_time_labeling " "consensus average_benchmark_agreement last_activity_time") From c6a8bec269459883c2dfcba70617b2ea8ca75143 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Sun, 7 Mar 2021 21:19:50 -0500 Subject: [PATCH 04/13] yapf --- labelbox/schema/data_row.py | 5 +- labelbox/schema/dataset.py | 9 +- labelbox/schema/webhook.py | 15 +- tests/integration/conftest.py | 305 ++++++++++-------- tests/integration/test_asset_metadata.py | 5 +- .../test_labeling_parameter_overrides.py | 4 +- tests/integration/test_project.py | 2 +- 7 files changed, 197 insertions(+), 148 deletions(-) diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index 632ab4165..ac391659c 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -33,7 +33,6 @@ class DataRow(DbObject, Updateable, BulkDeletable): labels = Relationship.ToMany("Label", True) metadata = Relationship.ToMany("AssetMetadata", False, "metadata") predictions = Relationship.ToMany("Prediction", False) - @staticmethod def bulk_delete(data_rows): @@ -62,7 +61,9 @@ def create_metadata(self, meta_type, meta_value): `AssetMetadata` DB object. """ if meta_type not in AssetMetadata.VALID_TYPES: - raise ValueError(f"metadata type must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}") + raise ValueError( + f"metadata type must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}" + ) meta_type_param = "metaType" meta_value_param = "metaValue" diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index 7aa5d6dd1..408585d22 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -164,7 +164,7 @@ def convert_item(item): task._user = user return task - def data_rows_for_external_id(self, external_id, limit = 10): + def data_rows_for_external_id(self, external_id, limit=10): """ Convenience method for getting a single `DataRow` belonging to this `Dataset` that has the given `external_id`. @@ -205,7 +205,10 @@ def data_row_for_external_id(self, external_id): in this `DataSet` with the given external ID, or if there are multiple `DataRows` for it. """ - data_rows = self.data_rows_for_external_id(external_id=external_id, limit = 2) + data_rows = self.data_rows_for_external_id(external_id=external_id, + limit=2) if len(data_rows) > 1: - logging.warn("More than one data_row has the provided external_id. Use function data_rows_for_external_id to fetch all") + logging.warn( + "More than one data_row has the provided external_id. Use function data_rows_for_external_id to fetch all" + ) return data_rows[0] diff --git a/labelbox/schema/webhook.py b/labelbox/schema/webhook.py index 1961cc039..700d5f9b2 100644 --- a/labelbox/schema/webhook.py +++ b/labelbox/schema/webhook.py @@ -5,6 +5,7 @@ logger = logging.getLogger(__name__) + class Webhook(DbObject, Updateable): """ Represents a server-side rule for sending notifications to a web-server whenever one of several predefined actions happens within a context of @@ -29,17 +30,13 @@ class Webhook(DbObject, Updateable): LABEL_UPDATED = "LABEL_UPDATED" LABEL_DELETED = "LABEL_DELETED" - REVIEW_CREATED ="REVIEW_CREATED" - REVIEW_UPDATED ="REVIEW_UPDATED" + REVIEW_CREATED = "REVIEW_CREATED" + REVIEW_UPDATED = "REVIEW_UPDATED" REVIEW_DELETED = "REVIEW_DELETED" SUPPORTED_TOPICS = { - LABEL_CREATED, - LABEL_UPDATED, - LABEL_DELETED, - REVIEW_CREATED, - REVIEW_UPDATED, - REVIEW_DELETED + LABEL_CREATED, LABEL_UPDATED, LABEL_DELETED, REVIEW_CREATED, + REVIEW_UPDATED, REVIEW_DELETED } updated_at = Field.DateTime("updated_at") @@ -86,7 +83,7 @@ def create(client, topics, url, secret, project): project = Relationship.ToOne("Project") def delete(self): - self.update(status = "INACTIVE") + self.update(status="INACTIVE") def update(self, topics=None, url=None, status=None): """ Updates this Webhook. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 409d130fe..a1f2f88d4 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -69,6 +69,7 @@ def client(environ: str): @pytest.fixture def rand_gen(): + def gen(field_type): if field_type is str: return "".join(ascii_letters[randint(0, @@ -124,33 +125,103 @@ def sample_video() -> str: assert os.path.exists(path_to_video) return path_to_video + @pytest.fixture def ontology(): - bbox_tool = {'required': False, 'name': 'bbox', 'tool': 'rectangle', 'color': '#a23030', 'classifications' : [ - { - 'required': False, 'instructions': 'nested', 'name': 'nested', 'type': 'radio', 'options': [ - { - 'label': 'radio_option_1', 'value': 'radio_value_1' - } - ] - } - ]} - polygon_tool = {'required': False, 'name': 'polygon', 'tool': 'polygon', 'color': '#FF34FF', 'classifications' : []} - polyline_tool = {'required': False, 'name': 'polyline', 'tool': 'line', 'color': '#FF4A46', 'classifications': []} - point_tool = {'required': False, 'name': 'point--', 'tool': 'point', 'color': '#008941', 'classifications': []} - entity_tool = {'required': False, 'name': 'entity--', 'tool': 'named-entity', 'color': '#006FA6', 'classifications': []} - segmentation_tool = {'required': False, 'name': 'segmentation--', 'tool': 'superpixel', 'color': '#A30059', 'classifications': []} - checklist = {'required': False, 'instructions': 'checklist', 'name': 'checklist', 'type': 'checklist', 'options': [{'label': 'option1', 'value': 'option1'}, {'label': 'option2', 'value': 'option2'}, {'label': 'optionN', 'value': 'optionn'}]} - free_form_text = {'required': False, 'instructions': 'text', 'name': 'text', 'type': 'text', 'options': []} + bbox_tool = { + 'required': + False, + 'name': + 'bbox', + 'tool': + 'rectangle', + 'color': + '#a23030', + 'classifications': [{ + 'required': False, + 'instructions': 'nested', + 'name': 'nested', + 'type': 'radio', + 'options': [{ + 'label': 'radio_option_1', + 'value': 'radio_value_1' + }] + }] + } + polygon_tool = { + 'required': False, + 'name': 'polygon', + 'tool': 'polygon', + 'color': '#FF34FF', + 'classifications': [] + } + polyline_tool = { + 'required': False, + 'name': 'polyline', + 'tool': 'line', + 'color': '#FF4A46', + 'classifications': [] + } + point_tool = { + 'required': False, + 'name': 'point--', + 'tool': 'point', + 'color': '#008941', + 'classifications': [] + } + entity_tool = { + 'required': False, + 'name': 'entity--', + 'tool': 'named-entity', + 'color': '#006FA6', + 'classifications': [] + } + segmentation_tool = { + 'required': False, + 'name': 'segmentation--', + 'tool': 'superpixel', + 'color': '#A30059', + 'classifications': [] + } + checklist = { + 'required': + False, + 'instructions': + 'checklist', + 'name': + 'checklist', + 'type': + 'checklist', + 'options': [{ + 'label': 'option1', + 'value': 'option1' + }, { + 'label': 'option2', + 'value': 'option2' + }, { + 'label': 'optionN', + 'value': 'optionn' + }] + } + free_form_text = { + 'required': False, + 'instructions': 'text', + 'name': 'text', + 'type': 'text', + 'options': [] + } ####Why does this break? #Adding radio buttons causes the whole ontology to be invalid.. - #radio_buttons = {'required': False, 'instructions': 'radio_tool', 'name': 'radio_tool', 'type': 'radio', 'options': [{'label': 'yes', 'value': 'yes'}, {'label': 'no', 'value': 'no'}]}, + #radio_buttons = {'required': False, 'instructions': 'radio_tool', 'name': 'radio_tool', 'type': 'radio', 'options': [{'label': 'yes', 'value': 'yes'}, {'label': 'no', 'value': 'no'}]}, #ontology = {"tools": [bbox_tool, polygon_tool, polyline_tool, point_tool, entity_tool, segmentation_tool], "classifications": [radio_buttons, checklist, free_form_text]} #### - tools = [bbox_tool, polygon_tool, polyline_tool, point_tool, entity_tool, segmentation_tool] + tools = [ + bbox_tool, polygon_tool, polyline_tool, point_tool, entity_tool, + segmentation_tool + ] classifications = [checklist, free_form_text] - return {"tools" : tools, "classifications" : classifications} + return {"tools": tools, "classifications": classifications} @pytest.fixture @@ -160,7 +231,7 @@ def configured_project(client, rand_gen, ontology): client.get_labeling_frontends( where=LabelingFrontend.name == "editor"))[0] project.setup(editor, ontology) - dataset = client.create_dataset(name = rand_gen(str)) + dataset = client.create_dataset(name=rand_gen(str)) for _ in range(len(ontology['tools']) + len(ontology['classifications'])): dataset.create_data_row(row_data=IMG_URL) project.datasets.connect(dataset) @@ -169,7 +240,6 @@ def configured_project(client, rand_gen, ontology): dataset.delete() - @pytest.fixture def prediction_id_mapping(configured_project): #Maps tool types to feature schema ids @@ -177,176 +247,159 @@ def prediction_id_mapping(configured_project): inferences = [] datarows = [d for d in list(configured_project.datasets())[0].data_rows()] result = {} - + for idx, tool in enumerate(ontology['tools'] + ontology['classifications']): if 'tool' in tool: tool_type = tool['tool'] else: tool_type = tool['type'] result[tool_type] = { - "uuid" : str(uuid.uuid4()), + "uuid": str(uuid.uuid4()), "schemaId": tool['featureSchemaId'], "dataRow": { "id": datarows[idx].uid, }, - 'tool' : tool + 'tool': tool } return result - + @pytest.fixture def polygon_inference(prediction_id_mapping): polygon = prediction_id_mapping['polygon'].copy() - polygon.update( -{ - "polygon": [{ - "x": 147.692, - "y": 118.154 - }, { - "x": 142.769, - "y": 404.923 - }, { - "x": 57.846, - "y": 318.769 - }, { - "x": 28.308, - "y": 169.846 - }] - } - ) + polygon.update({ + "polygon": [{ + "x": 147.692, + "y": 118.154 + }, { + "x": 142.769, + "y": 404.923 + }, { + "x": 57.846, + "y": 318.769 + }, { + "x": 28.308, + "y": 169.846 + }] + }) del polygon['tool'] return polygon + @pytest.fixture def rectangle_inference(prediction_id_mapping): rectangle = prediction_id_mapping['rectangle'].copy() - rectangle.update( {"bbox": { - "top": 48, - "left": 58, - "height": 865, - "width": 1512 - }, - 'classifications' : [{ - "schemaId" : rectangle['tool']['classifications'][0]['featureSchemaId'], - "answer": {"schemaId": rectangle['tool']['classifications'][0]['options'][0]['featureSchemaId'] } - }] - }) + rectangle.update({ + "bbox": { + "top": 48, + "left": 58, + "height": 865, + "width": 1512 + }, + 'classifications': [{ + "schemaId": + rectangle['tool']['classifications'][0]['featureSchemaId'], + "answer": { + "schemaId": + rectangle['tool']['classifications'][0]['options'][0] + ['featureSchemaId'] + } + }] + }) del rectangle['tool'] return rectangle + @pytest.fixture def line_inference(prediction_id_mapping): line = prediction_id_mapping['line'].copy() - line.update({ - "line": [{ - "x": 147.692, - "y": 118.154 - }, - { - "x": 150.692, - "y": 160.154 - }] - }) - del line['tool'] + line.update( + {"line": [{ + "x": 147.692, + "y": 118.154 + }, { + "x": 150.692, + "y": 160.154 + }]}) + del line['tool'] return line + @pytest.fixture def point_inference(prediction_id_mapping): point = prediction_id_mapping['point'].copy() - point.update({ - "point": { - "x": 147.692, - "y": 118.154 - }} - ) - del point['tool'] + point.update({"point": {"x": 147.692, "y": 118.154}}) + del point['tool'] return point + @pytest.fixture def entity_inference(prediction_id_mapping): entity = prediction_id_mapping['named-entity'].copy() - entity.update({"location" : { - "start" : 67, - "end" : 128 - }}) - del entity['tool'] + entity.update({"location": {"start": 67, "end": 128}}) + del entity['tool'] return entity + @pytest.fixture def segmentation_inference(prediction_id_mapping): segmentation = prediction_id_mapping['superpixel'].copy() - segmentation.update({'mask' : { - 'instanceURI' : "sampleuri", - 'colorRGB' : [0,0,0] - } - }) - del segmentation['tool'] - return segmentation + segmentation.update( + {'mask': { + 'instanceURI': "sampleuri", + 'colorRGB': [0, 0, 0] + }}) + del segmentation['tool'] + return segmentation + @pytest.fixture def checklist_inference(prediction_id_mapping): checklist = prediction_id_mapping['checklist'].copy() - checklist.update({'answers' : [ - { - 'schemaId' : checklist['tool']['options'][0]['featureSchemaId'] - } - ] - }) - del checklist['tool'] - return checklist + checklist.update({ + 'answers': [{ + 'schemaId': checklist['tool']['options'][0]['featureSchemaId'] + }] + }) + del checklist['tool'] + return checklist + @pytest.fixture def text_inference(prediction_id_mapping): text = prediction_id_mapping['text'].copy() - text.update({ - 'answer' : "free form text..." - }) + text.update({'answer': "free form text..."}) del text['tool'] - return text + return text @pytest.fixture def video_checklist_inference(prediction_id_mapping): checklist = prediction_id_mapping['checklist'].copy() - checklist.update({'answers' : [ - { - 'schemaId' : checklist['tool']['options'][0]['featureSchemaId'] - } - ] - }) - checklist.update({ - "frames": [ - { + 'answers': [{ + 'schemaId': checklist['tool']['options'][0]['featureSchemaId'] + }] + }) + + checklist.update( + {"frames": [{ "start": 7, "end": 13, - }, - { + }, { "start": 18, "end": 19, - } - ]}) - del checklist['tool'] - return checklist - - + }]}) + del checklist['tool'] + return checklist @pytest.fixture -def predictions(polygon_inference, - rectangle_inference, - line_inference, - entity_inference, - segmentation_inference, - checklist_inference, - text_inference): +def predictions(polygon_inference, rectangle_inference, line_inference, + entity_inference, segmentation_inference, checklist_inference, + text_inference): return [ - polygon_inference, - rectangle_inference, - line_inference, - entity_inference, - segmentation_inference, - checklist_inference, + polygon_inference, rectangle_inference, line_inference, + entity_inference, segmentation_inference, checklist_inference, text_inference ] - \ No newline at end of file diff --git a/tests/integration/test_asset_metadata.py b/tests/integration/test_asset_metadata.py index 2f3c64296..7ed7048d1 100644 --- a/tests/integration/test_asset_metadata.py +++ b/tests/integration/test_asset_metadata.py @@ -21,7 +21,7 @@ def test_asset_metadata_crud(project, dataset, rand_gen): with pytest.raises(ValueError): data_row.create_metadata("NOT_SUPPORTED_TYPE", "Value") - + # Check that filtering and sorting is prettily disabled with pytest.raises(InvalidQueryError) as exc_info: data_row.metadata(where=AssetMetadata.meta_value == "x") @@ -31,6 +31,3 @@ def test_asset_metadata_crud(project, dataset, rand_gen): data_row.metadata(order_by=AssetMetadata.meta_value.asc) assert exc_info.value.message == \ "Relationship DataRow.metadata doesn't support sorting" - - - diff --git a/tests/integration/test_labeling_parameter_overrides.py b/tests/integration/test_labeling_parameter_overrides.py index fd4c1adfd..054fcfa54 100644 --- a/tests/integration/test_labeling_parameter_overrides.py +++ b/tests/integration/test_labeling_parameter_overrides.py @@ -34,11 +34,10 @@ def test_labeling_parameter_overrides(project, rand_gen): # currently this doesn't work so the count remains 3 assert len(list(project.labeling_parameter_overrides())) == 1 - with pytest.raises(TypeError): data = [(data_rows[12], "yoo", 3)] project.set_labeling_parameter_overrides(data) - + with pytest.raises(TypeError): data = [(data_rows[12], 3, "yoo")] project.set_labeling_parameter_overrides(data) @@ -56,4 +55,3 @@ def test_labeling_parameter_overrides(project, rand_gen): project.set_labeling_parameter_overrides(data) dataset.delete() - diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index d57ed4e67..b2a48713e 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -60,7 +60,7 @@ def test_upsert_review_queue(project): with pytest.raises(ValueError): project.upsert_review_queue(1.001) - + with pytest.raises(ValueError): project.upsert_review_queue(-0.001) From 8d16c834dc53358f9708f095f0c6522f43f80941 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Sun, 7 Mar 2021 21:22:15 -0500 Subject: [PATCH 05/13] yapf --- labelbox/schema/asset_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/labelbox/schema/asset_metadata.py b/labelbox/schema/asset_metadata.py index 91e2eca5f..38f24a44b 100644 --- a/labelbox/schema/asset_metadata.py +++ b/labelbox/schema/asset_metadata.py @@ -9,12 +9,12 @@ class AssetMetadata(DbObject): meta_type (str): IMAGE, VIDEO, TEXT, or IMAGE_OVERLAY meta_value (str): URL to an external file or a string of text """ - + VIDEO = "VIDEO" IMAGE = "IMAGE" - TEXT= "TEXT" + TEXT = "TEXT" IMAGE_OVERLAY = "IMAGE_OVERLAY" - + SUPPORTED_TYPES = {VIDEO, IMAGE, TEXT, IMAGE_OVERLAY} meta_type = Field.String("meta_type") From acf77306af0ea923b6c6dc5ee5fd5f861744d7eb Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Sun, 7 Mar 2021 21:26:39 -0500 Subject: [PATCH 06/13] small bug --- labelbox/schema/data_row.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index ac391659c..6911cd2cf 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -55,14 +55,14 @@ def create_metadata(self, meta_type, meta_value): Args: meta_type (str): Asset metadata type, must be one of: - VIDEO, IMAGE, TEXT. + VIDEO, IMAGE, TEXT, IMAGE_OVERLAY (AssetMetadata.SUPPORTED_TYPES) meta_value (str): Asset metadata value. Returns: `AssetMetadata` DB object. """ - if meta_type not in AssetMetadata.VALID_TYPES: + if meta_type not in AssetMetadata.SUPPORTED_TYPES: raise ValueError( - f"metadata type must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}" + f"metadata type must be one of {AssetMetadata.SUPPORTED_TYPES}. Found {meta_type}" ) meta_type_param = "metaType" From 5b238b1170da67f14a9ca04ce0fae474d39c348d Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Sun, 7 Mar 2021 21:48:31 -0500 Subject: [PATCH 07/13] fix test --- tests/integration/test_dataset.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 701c88701..369fefb06 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -75,10 +75,8 @@ def test_get_data_row_for_external_id(dataset, rand_gen): assert found.uid == data_row.uid assert found.external_id == external_id - second = dataset.create_data_row(row_data=IMG_URL, external_id=external_id) - - with pytest.raises(ResourceNotFoundError): - data_row = dataset.data_row_for_external_id(external_id) + dataset.create_data_row(row_data=IMG_URL, external_id=external_id) + assert len(dataset.data_rows_for_external_id(external_id)) == 2 def test_upload_video_file(dataset, sample_video: str) -> None: From cabc82c638f6a8182faadc4c9ba9eea47a74a28f Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Sun, 7 Mar 2021 22:12:53 -0500 Subject: [PATCH 08/13] return extend_reservations --- labelbox/schema/project.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index f0921a8f6..f55cd967d 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -369,6 +369,24 @@ def upsert_review_queue(self, quota_factor): quota_param: quota_factor }) + def extend_reservations(self, queue_type): + """ Extends all the current reservations for the current user on the given + queue type. + Args: + queue_type (str): Either "LabelingQueue" or "ReviewQueue" + Returns: + int, the number of reservations that were extended. + """ + if queue_type not in ("LabelingQueue", "ReviewQueue"): + raise InvalidQueryError("Unsupported queue type: %s" % queue_type) + + id_param = "projectId" + query_str = """mutation ExtendReservationsPyApi($%s: ID!){ + extendReservations(projectId:$%s queueType:%s)}""" % ( + id_param, id_param, queue_type) + res = self.client.execute(query_str, {id_param: self.uid}) + return res["extendReservations"] + def create_prediction_model(self, name, version): """ Creates a PredictionModel connected to a Legacy Editor Project. From 978f983609d3e55580d61d541ae846c43cc5f2b1 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Wed, 10 Mar 2021 18:19:51 -0500 Subject: [PATCH 09/13] webhook validation. Update tests --- labelbox/schema/asset_metadata.py | 15 ++-- labelbox/schema/data_row.py | 9 +- labelbox/schema/dataset.py | 1 + labelbox/schema/project.py | 68 +++++++-------- labelbox/schema/webhook.py | 86 +++++++++++++------ tests/integration/test_asset_metadata.py | 4 +- .../test_labeling_parameter_overrides.py | 28 ++++-- tests/integration/test_project.py | 10 ++- tests/integration/test_webhook.py | 15 ++++ 9 files changed, 152 insertions(+), 84 deletions(-) diff --git a/labelbox/schema/asset_metadata.py b/labelbox/schema/asset_metadata.py index 38f24a44b..73036c449 100644 --- a/labelbox/schema/asset_metadata.py +++ b/labelbox/schema/asset_metadata.py @@ -1,3 +1,5 @@ +from enum import Enum + from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field @@ -10,12 +12,15 @@ class AssetMetadata(DbObject): meta_value (str): URL to an external file or a string of text """ - VIDEO = "VIDEO" - IMAGE = "IMAGE" - TEXT = "TEXT" - IMAGE_OVERLAY = "IMAGE_OVERLAY" + class MetaType(Enum): + VIDEO = "VIDEO" + IMAGE = "IMAGE" + TEXT = "TEXT" + IMAGE_OVERLAY = "IMAGE_OVERLAY" - SUPPORTED_TYPES = {VIDEO, IMAGE, TEXT, IMAGE_OVERLAY} + #For backwards compatibility + for topic in MetaType: + vars()[topic.name] = topic.value meta_type = Field.String("meta_type") meta_value = Field.String("meta_value") diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index 6911cd2cf..044caafe5 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -55,14 +55,17 @@ def create_metadata(self, meta_type, meta_value): Args: meta_type (str): Asset metadata type, must be one of: - VIDEO, IMAGE, TEXT, IMAGE_OVERLAY (AssetMetadata.SUPPORTED_TYPES) + VIDEO, IMAGE, TEXT, IMAGE_OVERLAY (AssetMetadata.MetaType) meta_value (str): Asset metadata value. Returns: `AssetMetadata` DB object. + Raises: + ValueError: meta_type must be one of the supported types. """ - if meta_type not in AssetMetadata.SUPPORTED_TYPES: + supported_meta_types = [x.value for x in AssetMetadata.MetaType] + if meta_type not in supported_meta_types: raise ValueError( - f"metadata type must be one of {AssetMetadata.SUPPORTED_TYPES}. Found {meta_type}" + f"metadata type must be one of {supported_meta_types}. Found {meta_type}" ) meta_type_param = "metaType" diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index 408585d22..6b67e88e5 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -170,6 +170,7 @@ def data_rows_for_external_id(self, external_id, limit=10): Args: external_id (str): External ID of the sought `DataRow`. + limit (int): The maximum number of data rows to return for the given external_id Returns: A single `DataRow` with the given ID. diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index f408c7f4f..d5d572d35 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -1,15 +1,15 @@ -from collections import namedtuple -from datetime import datetime, timezone import json -from labelbox.schema.data_row import DataRow +import time import logging +from collections import namedtuple +from datetime import datetime, timezone from pathlib import Path -import time from typing import Dict, List, Union, Iterable from urllib.parse import urlparse from labelbox import utils from labelbox.schema.bulk_import_request import BulkImportRequest +from labelbox.schema.data_row import DataRow from labelbox.exceptions import InvalidQueryError from labelbox.orm import query from labelbox.orm.db_object import DbObject, Updateable, Deletable @@ -320,50 +320,28 @@ def validate_labeling_parameter_overrides(self, data): for idx, row in enumerate(data): if len(row) != 3: raise TypeError( - f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items" + f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items. Index: {idx}" ) data_row, priority, num_labels = row if not isinstance(data_row, DataRow): raise TypeError( - f"Datarow should be be of type DataRow. Found {data_row}") + f"Datarow should be be of type DataRow. Found {type(data_row)}. Index: {idx}" + ) - for name, value in [["priority", priority], + for name, value in [["Priority", priority], ["Number of labels", num_labels]]: if not isinstance(value, int): raise TypeError( - f"{name} must be an int. Found {type(value)} for data_row {data_row}" + f"{name} must be an int. Found {type(value)} for data_row {data_row}. Index: {idx}" ) if value < 1: raise ValueError( - f"{name} must be greater than 0 for data_row {data_row}" + f"{name} must be greater than 0 for data_row {data_row}. Index: {idx}" ) def set_labeling_parameter_overrides(self, data): """ Adds labeling parameter overrides to this project. - - Priority: - * data will be labeled in priority order - - lower numbers labeled first - - Minimum priority is 1. - * Priority is not the queue position. - - The position is determined by the relative priority. - - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] - will be assigned in the following order: [data_row_2, data_row_1, data_row_3] - * datarows with parameter overrides will appear before datarows without overrides - * The priority only effects items in the queue - - Assigning a priority will not automatically add the item back into the queue - - Number of labels: - * The number times a data row should be labeled - * This will create duplicates in a project - * The queue will never assign the same datarow to a labeler more than once - - if the number of labels is greater than the number of labelers working on a project then - the extra items will get stuck in the queue (thsi can be fixed by removing the override at any time). - * This can add items to the queue (even if they have already been labeled) - - New copies will only be assigned to members who have not labeled that same datarow before. - * Setting this to 1 will result in the default behavior (no duplicates) - - + See information on priority here: https://docs.labelbox.com/en/configure-editor/queue-system#reservation-system @@ -372,7 +350,29 @@ def set_labeling_parameter_overrides(self, data): Args: data (iterable): An iterable of tuples. Each tuple must contain - (Union[DataRow, datarow_id], priority, numberOfLabels) for the new override. + (DataRow, priority, number_of_labels) for the new override. + Priority: + * Data will be labeled in priority order. + - Lower number priority is labeled first. + - Minimum priority is 1. + * Priority is not the queue position. + - The position is determined by the relative priority. + - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] + will be assigned in the following order: [data_row_2, data_row_1, data_row_3] + * Datarows with parameter overrides will appear before datarows without overrides. + * The priority only effects items in the queue. + - Assigning a priority will not automatically add the item back into the queue. + Number of labels: + * The number times a data row should be labeled. + * This will create duplicate data rows in a project (one for each number of labels). + * Data rows will be sent to the queue (even if they have already been labeled). + - New copies will only be assigned to members who have not labeled that same datarow before. + - Already labeled duplicates will not be sent back to the queue. + * The queue will never assign the same datarow to a single labeler more than once. + - If the number of labels is greater than the number of labelers working on a project then + the extra items will remain in the queue (this can be fixed by removing the override at any time). + + * Setting this to 1 will result in the default behavior (no duplicates). Returns: bool, indicates if the operation was a success. """ diff --git a/labelbox/schema/webhook.py b/labelbox/schema/webhook.py index 700d5f9b2..8fe0cebd0 100644 --- a/labelbox/schema/webhook.py +++ b/labelbox/schema/webhook.py @@ -1,4 +1,7 @@ import logging +from enum import Enum +from typing import Iterable, List + from labelbox.orm import query from labelbox.orm.db_object import DbObject, Updateable from labelbox.orm.model import Entity, Field, Relationship @@ -16,28 +19,30 @@ class Webhook(DbObject, Updateable): created_at (datetime) url (str) topics (str): LABEL_CREATED, LABEL_UPDATED, LABEL_DELETED + REVIEW_CREATED, REVIEW_UPDATED, REVIEW_DELETED status (str): ACTIVE, INACTIVE, REVOKED """ - # Status - ACTIVE = "ACTIVE" - INACTIVE = "INACTIVE" - REVOKED = "REVOKED" + class WebhookStatus(Enum): + ACTIVE = "ACTIVE" + INACTIVE = "INACTIVE" + REVOKED = "REVOKED" - # Topic - LABEL_CREATED = "LABEL_CREATED" - LABEL_UPDATED = "LABEL_UPDATED" - LABEL_DELETED = "LABEL_DELETED" + class WebhookTopic(Enum): + LABEL_CREATED = "LABEL_CREATED" + LABEL_UPDATED = "LABEL_UPDATED" + LABEL_DELETED = "LABEL_DELETED" + REVIEW_CREATED = "REVIEW_CREATED" + REVIEW_UPDATED = "REVIEW_UPDATED" + REVIEW_DELETED = "REVIEW_DELETED" - REVIEW_CREATED = "REVIEW_CREATED" - REVIEW_UPDATED = "REVIEW_UPDATED" - REVIEW_DELETED = "REVIEW_DELETED" + #For backwards compatibility + for topic in WebhookStatus: + vars()[topic.name] = topic.value - SUPPORTED_TOPICS = { - LABEL_CREATED, LABEL_UPDATED, LABEL_DELETED, REVIEW_CREATED, - REVIEW_UPDATED, REVIEW_DELETED - } + for status in WebhookTopic: + vars()[status.name] = status.value updated_at = Field.DateTime("updated_at") created_at = Field.DateTime("created_at") @@ -45,6 +50,10 @@ class Webhook(DbObject, Updateable): topics = Field.String("topics") status = Field.String("status") + created_by = Relationship.ToOne("User", False, "created_by") + organization = Relationship.ToOne("Organization") + project = Relationship.ToOne("Project") + @staticmethod def create(client, topics, url, secret, project): """ Creates a Webhook. @@ -53,7 +62,7 @@ def create(client, topics, url, secret, project): client (Client): The Labelbox client used to connect to the server. topics (list of str): A list of topics this Webhook should - get notifications for. Must be one of Webhook.SUPPORTED_TOPICS + get notifications for. Must be one of Webhook.WebhookTopic url (str): The URL to which notifications should be sent by the Labelbox server. secret (str): A secret key used for signing notifications. @@ -63,10 +72,14 @@ def create(client, topics, url, secret, project): Returns: A newly created Webhook. + Raises: + ValueError: If the topic is not one of WebhookTopic or status is not one of WebhookStatus + Information on configuring your server can be found here (this is where the url points to and the secret is set). https://docs.labelbox.com/en/configure-editor/webhooks-setup#setup-steps - + """ + Webhook.validate_topics(topics) project_str = "" if project is None \ else ("project:{id:\"%s\"}," % project.uid) @@ -78,30 +91,47 @@ def create(client, topics, url, secret, project): return Webhook(client, client.execute(query_str)["createWebhook"]) - created_by = Relationship.ToOne("User", False, "created_by") - organization = Relationship.ToOne("Organization") - project = Relationship.ToOne("Project") + @staticmethod + def validate_topics(topics: List["Webhook.WebhookTopic"]): + if not isinstance(topics, list): + raise TypeError( + f"Topics must be List[Webhook.WebhookTopic]. Found `{topics}`") + + for topic in topics: + Webhook.validate_value(topic, Webhook.WebhookTopic) + + @staticmethod + def validate_value(value, enum): + supported_values = [x.value for x in enum] + if value not in supported_values: + raise ValueError( + f"Value `{value}` does not exist in supported values. Expected one of {supported_values}" + ) def delete(self): - self.update(status="INACTIVE") + self.update(status=self.WebhookStatus.INACTIVE) def update(self, topics=None, url=None, status=None): - """ Updates this Webhook. + """ Updates the Webhook. Args: - topics (list of str): The new topics value, optional. - url (str): The new URL value, optional. - status (str): The new status value, optional. + topics (List[str]): The new topics. + url (str): The new URL value. + status (str): The new status. If values are set to None then there are no updates made to that field. - The following code will delete the webhook. - >>> self.update(status = Webhook.INACTIVE) - """ # Webhook has a custom `update` function due to custom types # in `status` and `topics` fields. + + if topics is not None: + self.validate_topics(topics) + + if status is not None: + self.validate_value(status, self.WebhookStatus) + topics_str = "" if topics is None \ else "topics: {set: [%s]}" % " ".join(topics) url_str = "" if url is None else "url: \"%s\"" % url diff --git a/tests/integration/test_asset_metadata.py b/tests/integration/test_asset_metadata.py index 7ed7048d1..0c59663cd 100644 --- a/tests/integration/test_asset_metadata.py +++ b/tests/integration/test_asset_metadata.py @@ -19,8 +19,10 @@ def test_asset_metadata_crud(project, dataset, rand_gen): assert asset.meta_value == "Value" assert len(list(data_row.metadata())) == 1 - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: data_row.create_metadata("NOT_SUPPORTED_TYPE", "Value") + assert str(exc_info.value) == \ + f"metadata type must be one of {[x.value for x in AssetMetadata.MetaType]}. Found NOT_SUPPORTED_TYPE" # Check that filtering and sorting is prettily disabled with pytest.raises(InvalidQueryError) as exc_info: diff --git a/tests/integration/test_labeling_parameter_overrides.py b/tests/integration/test_labeling_parameter_overrides.py index 054fcfa54..caaf773c4 100644 --- a/tests/integration/test_labeling_parameter_overrides.py +++ b/tests/integration/test_labeling_parameter_overrides.py @@ -34,24 +34,34 @@ def test_labeling_parameter_overrides(project, rand_gen): # currently this doesn't work so the count remains 3 assert len(list(project.labeling_parameter_overrides())) == 1 - with pytest.raises(TypeError): - data = [(data_rows[12], "yoo", 3)] + with pytest.raises(TypeError) as exc_info: + data = [(data_rows[12], "a_string", 3)] project.set_labeling_parameter_overrides(data) + assert str(exc_info.value) == \ + f"Priority must be an int. Found for data_row {data_rows[12]}. Index: 0" - with pytest.raises(TypeError): - data = [(data_rows[12], 3, "yoo")] + with pytest.raises(TypeError) as exc_info: + data = [(data_rows[12], 3, "a_string")] project.set_labeling_parameter_overrides(data) + assert str(exc_info.value) == \ + f"Number of labels must be an int. Found for data_row {data_rows[12]}. Index: 0" - with pytest.raises(TypeError): + with pytest.raises(TypeError) as exc_info: data = [(data_rows[12].uid, 1, 3)] project.set_labeling_parameter_overrides(data) + assert str(exc_info.value) == \ + "Datarow should be be of type DataRow. Found . Index: 0" - with pytest.raises(TypeError): - data = [(data_rows[12].uid, 0, 3)] + with pytest.raises(ValueError) as exc_info: + data = [(data_rows[12], 0, 3)] project.set_labeling_parameter_overrides(data) + assert str(exc_info.value) == \ + f"Priority must be greater than 0 for data_row {data_rows[12]}. Index: 0" - with pytest.raises(TypeError): - data = [(data_rows[12].uid, 1, 0)] + with pytest.raises(ValueError) as exc_info: + data = [(data_rows[12], 1, 0)] project.set_labeling_parameter_overrides(data) + assert str(exc_info.value) == \ + f"Number of labels must be greater than 0 for data_row {data_rows[12]}. Index: 0" dataset.delete() diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 04079e945..a49cf7070 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -60,11 +60,13 @@ def test_project_filtering(client, rand_gen): def test_upsert_review_queue(project): project.upsert_review_queue(0.6) - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: project.upsert_review_queue(1.001) + assert str(exc_info.value) == "Quota factor must be in the range of [0,1]" - with pytest.raises(ValueError): + with pytest.raises(ValueError) as exc_info: project.upsert_review_queue(-0.001) + assert str(exc_info.value) == "Quota factor must be in the range of [0,1]" def test_extend_reservations(project): @@ -95,6 +97,6 @@ def test_attach_instructions(client, project): list(project.labeling_frontend_options()) [-1].customization_options).get('projectInstructions') is not None - with pytest.raises(ValueError) as execinfo: + with pytest.raises(ValueError) as exc_info: project.upsert_instructions('/tmp/file.invalid_file_extension') - assert "instructions_file must end with one of" in str(execinfo.value) + assert "instructions_file must end with one of" in str(exc_info.value) diff --git a/tests/integration/test_webhook.py b/tests/integration/test_webhook.py index ebe32341e..e281f5ea2 100644 --- a/tests/integration/test_webhook.py +++ b/tests/integration/test_webhook.py @@ -21,3 +21,18 @@ def test_webhook_create_update(project, rand_gen): webhook.update(status=Webhook.REVOKED, topics=[Webhook.LABEL_UPDATED]) assert webhook.topics == [Webhook.LABEL_UPDATED] assert webhook.status == Webhook.REVOKED + + with pytest.raises(ValueError) as exc_info: + webhook.update(status="invalid..") + assert str(exc_info.value) == \ + f"Value `invalid..` does not exist in supported values. Expected one of {[x.value for x in Webhook.WebhookStatus]}" + + with pytest.raises(ValueError) as exc_info: + webhook.update(topics=["invalid.."]) + assert str(exc_info.value) == \ + f"Value `invalid..` does not exist in supported values. Expected one of {[x.value for x in Webhook.WebhookTopic]}" + + with pytest.raises(TypeError) as exc_info: + webhook.update(topics="invalid..") + assert str(exc_info.value) == \ + "Topics must be List[Webhook.WebhookTopic]. Found `invalid..`" From e356ee975aa53d531aa79421a828e2f330419184 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Wed, 10 Mar 2021 18:32:47 -0500 Subject: [PATCH 10/13] add type hints, fix typos --- labelbox/schema/project.py | 11 +++++------ labelbox/schema/webhook.py | 24 ++++++++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index d5d572d35..a36763f6c 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -351,9 +351,10 @@ def set_labeling_parameter_overrides(self, data): Args: data (iterable): An iterable of tuples. Each tuple must contain (DataRow, priority, number_of_labels) for the new override. + Priority: * Data will be labeled in priority order. - - Lower number priority is labeled first. + - A lower number priority is labeled first. - Minimum priority is 1. * Priority is not the queue position. - The position is determined by the relative priority. @@ -363,15 +364,13 @@ def set_labeling_parameter_overrides(self, data): * The priority only effects items in the queue. - Assigning a priority will not automatically add the item back into the queue. Number of labels: - * The number times a data row should be labeled. - * This will create duplicate data rows in a project (one for each number of labels). - * Data rows will be sent to the queue (even if they have already been labeled). - - New copies will only be assigned to members who have not labeled that same datarow before. + * The number of times a data row should be labeled. + - Creates duplicate data rows in a project (one for each number of labels). + * New duplicated data rows will be added to the queue. - Already labeled duplicates will not be sent back to the queue. * The queue will never assign the same datarow to a single labeler more than once. - If the number of labels is greater than the number of labelers working on a project then the extra items will remain in the queue (this can be fixed by removing the override at any time). - * Setting this to 1 will result in the default behavior (no duplicates). Returns: bool, indicates if the operation was a success. diff --git a/labelbox/schema/webhook.py b/labelbox/schema/webhook.py index 8fe0cebd0..36f025f49 100644 --- a/labelbox/schema/webhook.py +++ b/labelbox/schema/webhook.py @@ -1,10 +1,11 @@ import logging from enum import Enum -from typing import Iterable, List +from typing import List, Optional, Union from labelbox.orm import query from labelbox.orm.db_object import DbObject, Updateable from labelbox.orm.model import Entity, Field, Relationship +from labelbox import Client, Project logger = logging.getLogger(__name__) @@ -55,7 +56,8 @@ class WebhookTopic(Enum): project = Relationship.ToOne("Project") @staticmethod - def create(client, topics, url, secret, project): + def create(client: Client, topics: List[WebhookTopic], url: str, + secret: str, project: Project): """ Creates a Webhook. Args: @@ -92,7 +94,7 @@ def create(client, topics, url, secret, project): return Webhook(client, client.execute(query_str)["createWebhook"]) @staticmethod - def validate_topics(topics: List["Webhook.WebhookTopic"]): + def validate_topics(topics: List[WebhookTopic]): if not isinstance(topics, list): raise TypeError( f"Topics must be List[Webhook.WebhookTopic]. Found `{topics}`") @@ -101,7 +103,7 @@ def validate_topics(topics: List["Webhook.WebhookTopic"]): Webhook.validate_value(topic, Webhook.WebhookTopic) @staticmethod - def validate_value(value, enum): + def validate_value(value: str, enum: Union[WebhookStatus, WebhookTopic]): supported_values = [x.value for x in enum] if value not in supported_values: raise ValueError( @@ -109,15 +111,21 @@ def validate_value(value, enum): ) def delete(self): + """ + Deletes the webhook + """ self.update(status=self.WebhookStatus.INACTIVE) - def update(self, topics=None, url=None, status=None): + def update(self, + topics: Optional[List[WebhookTopic]] = None, + url: Optional[str] = None, + status: Optional[WebhookStatus] = None): """ Updates the Webhook. Args: - topics (List[str]): The new topics. - url (str): The new URL value. - status (str): The new status. + topics (Optional[List[WebhookTopic]]): The new topics. + url Optional[str): The new URL value. + status (Optional[WebhookStatus]): The new status. If values are set to None then there are no updates made to that field. From 43a501c78eff77407a8626bcb79b50e4efc11b80 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Wed, 10 Mar 2021 21:12:34 -0500 Subject: [PATCH 11/13] requested changes --- labelbox/schema/asset_metadata.py | 2 +- labelbox/schema/data_row.py | 10 ++++-- labelbox/schema/dataset.py | 2 +- labelbox/schema/project.py | 8 ++--- labelbox/schema/webhook.py | 45 +++++++++++------------- tests/integration/test_asset_metadata.py | 3 +- tests/integration/test_webhook.py | 8 +++-- 7 files changed, 40 insertions(+), 38 deletions(-) diff --git a/labelbox/schema/asset_metadata.py b/labelbox/schema/asset_metadata.py index 73036c449..5a326d276 100644 --- a/labelbox/schema/asset_metadata.py +++ b/labelbox/schema/asset_metadata.py @@ -18,7 +18,7 @@ class MetaType(Enum): TEXT = "TEXT" IMAGE_OVERLAY = "IMAGE_OVERLAY" - #For backwards compatibility + # For backwards compatibility for topic in MetaType: vars()[topic.name] = topic.value diff --git a/labelbox/schema/data_row.py b/labelbox/schema/data_row.py index 044caafe5..57afa60c8 100644 --- a/labelbox/schema/data_row.py +++ b/labelbox/schema/data_row.py @@ -34,6 +34,10 @@ class DataRow(DbObject, Updateable, BulkDeletable): metadata = Relationship.ToMany("AssetMetadata", False, "metadata") predictions = Relationship.ToMany("Prediction", False) + supported_meta_types = { + meta_type.value for meta_type in AssetMetadata.MetaType + } + @staticmethod def bulk_delete(data_rows): """ Deletes all the given DataRows. @@ -62,10 +66,10 @@ def create_metadata(self, meta_type, meta_value): Raises: ValueError: meta_type must be one of the supported types. """ - supported_meta_types = [x.value for x in AssetMetadata.MetaType] - if meta_type not in supported_meta_types: + + if meta_type not in self.supported_meta_types: raise ValueError( - f"metadata type must be one of {supported_meta_types}. Found {meta_type}" + f"meta_type must be one of {self.supported_meta_types}. Found {meta_type}" ) meta_type_param = "metaType" diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index 6b67e88e5..c2b944c16 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -210,6 +210,6 @@ def data_row_for_external_id(self, external_id): limit=2) if len(data_rows) > 1: logging.warn( - "More than one data_row has the provided external_id. Use function data_rows_for_external_id to fetch all" + f"More than one data_row has the provided external_id : `{external_id}`. Use function data_rows_for_external_id to fetch all" ) return data_rows[0] diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index a36763f6c..ade8f3ac0 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -90,7 +90,7 @@ def create_label(self, **kwargs): # about them. At the same time they're connected to a Label at # label creation in a non-standard way (connect via name). logger.warning( - "This function is deprecated and is not compatible with the new editor." + "`create_label` is deprecated and is not compatible with the new editor." ) Label = Entity.Label @@ -417,7 +417,7 @@ def upsert_review_queue(self, quota_factor): to reinitiate. Between 0 and 1. """ - if (quota_factor > 1.) or (quota_factor < 0.): + if not 0. < quota_factor < 1.: raise ValueError("Quota factor must be in the range of [0,1]") id_param = "projectId" @@ -460,7 +460,7 @@ def create_prediction_model(self, name, version): """ logger.warning( - "This function is deprecated and is not compatible with the new editor." + "`create_prediction_model` is deprecated and is not compatible with the new editor." ) PM = Entity.PredictionModel @@ -489,7 +489,7 @@ def create_prediction(self, label, data_row, prediction_model=None): None. """ logger.warning( - "This function is deprecated and is not compatible with the new editor." + "`create_prediction` is deprecated and is not compatible with the new editor." ) if prediction_model is None: diff --git a/labelbox/schema/webhook.py b/labelbox/schema/webhook.py index 36f025f49..5091dec3e 100644 --- a/labelbox/schema/webhook.py +++ b/labelbox/schema/webhook.py @@ -1,11 +1,10 @@ import logging from enum import Enum -from typing import List, Optional, Union +from typing import Iterable, List from labelbox.orm import query from labelbox.orm.db_object import DbObject, Updateable from labelbox.orm.model import Entity, Field, Relationship -from labelbox import Client, Project logger = logging.getLogger(__name__) @@ -25,12 +24,12 @@ class Webhook(DbObject, Updateable): """ - class WebhookStatus(Enum): + class Status(Enum): ACTIVE = "ACTIVE" INACTIVE = "INACTIVE" REVOKED = "REVOKED" - class WebhookTopic(Enum): + class Topic(Enum): LABEL_CREATED = "LABEL_CREATED" LABEL_UPDATED = "LABEL_UPDATED" LABEL_DELETED = "LABEL_DELETED" @@ -38,11 +37,11 @@ class WebhookTopic(Enum): REVIEW_UPDATED = "REVIEW_UPDATED" REVIEW_DELETED = "REVIEW_DELETED" - #For backwards compatibility - for topic in WebhookStatus: + # For backwards compatibility + for topic in Status: vars()[topic.name] = topic.value - for status in WebhookTopic: + for status in Topic: vars()[status.name] = status.value updated_at = Field.DateTime("updated_at") @@ -56,15 +55,14 @@ class WebhookTopic(Enum): project = Relationship.ToOne("Project") @staticmethod - def create(client: Client, topics: List[WebhookTopic], url: str, - secret: str, project: Project): + def create(client, topics, url, secret, project): """ Creates a Webhook. Args: client (Client): The Labelbox client used to connect to the server. topics (list of str): A list of topics this Webhook should - get notifications for. Must be one of Webhook.WebhookTopic + get notifications for. Must be one of Webhook.Topic url (str): The URL to which notifications should be sent by the Labelbox server. secret (str): A secret key used for signing notifications. @@ -75,7 +73,7 @@ def create(client: Client, topics: List[WebhookTopic], url: str, A newly created Webhook. Raises: - ValueError: If the topic is not one of WebhookTopic or status is not one of WebhookStatus + ValueError: If the topic is not one of Topic or status is not one of Status Information on configuring your server can be found here (this is where the url points to and the secret is set). https://docs.labelbox.com/en/configure-editor/webhooks-setup#setup-steps @@ -94,17 +92,17 @@ def create(client: Client, topics: List[WebhookTopic], url: str, return Webhook(client, client.execute(query_str)["createWebhook"]) @staticmethod - def validate_topics(topics: List[WebhookTopic]): - if not isinstance(topics, list): + def validate_topics(topics): + if isinstance(topics, str) or not isinstance(topics, Iterable): raise TypeError( - f"Topics must be List[Webhook.WebhookTopic]. Found `{topics}`") + f"Topics must be List[Webhook.Topic]. Found `{topics}`") for topic in topics: - Webhook.validate_value(topic, Webhook.WebhookTopic) + Webhook.validate_value(topic, Webhook.Topic) @staticmethod - def validate_value(value: str, enum: Union[WebhookStatus, WebhookTopic]): - supported_values = [x.value for x in enum] + def validate_value(value, enum): + supported_values = {item.value for item in enum} if value not in supported_values: raise ValueError( f"Value `{value}` does not exist in supported values. Expected one of {supported_values}" @@ -114,18 +112,15 @@ def delete(self): """ Deletes the webhook """ - self.update(status=self.WebhookStatus.INACTIVE) + self.update(status=self.Status.INACTIVE) - def update(self, - topics: Optional[List[WebhookTopic]] = None, - url: Optional[str] = None, - status: Optional[WebhookStatus] = None): + def update(self, topics=None, url=None, status=None): """ Updates the Webhook. Args: - topics (Optional[List[WebhookTopic]]): The new topics. + topics (Optional[List[Topic]]): The new topics. url Optional[str): The new URL value. - status (Optional[WebhookStatus]): The new status. + status (Optional[Status]): The new status. If values are set to None then there are no updates made to that field. @@ -138,7 +133,7 @@ def update(self, self.validate_topics(topics) if status is not None: - self.validate_value(status, self.WebhookStatus) + self.validate_value(status, self.Status) topics_str = "" if topics is None \ else "topics: {set: [%s]}" % " ".join(topics) diff --git a/tests/integration/test_asset_metadata.py b/tests/integration/test_asset_metadata.py index 0c59663cd..295ef8d4d 100644 --- a/tests/integration/test_asset_metadata.py +++ b/tests/integration/test_asset_metadata.py @@ -21,8 +21,9 @@ def test_asset_metadata_crud(project, dataset, rand_gen): with pytest.raises(ValueError) as exc_info: data_row.create_metadata("NOT_SUPPORTED_TYPE", "Value") + expected_types = {item.value for item in AssetMetadata.MetaType} assert str(exc_info.value) == \ - f"metadata type must be one of {[x.value for x in AssetMetadata.MetaType]}. Found NOT_SUPPORTED_TYPE" + f"meta_type must be one of {expected_types}. Found NOT_SUPPORTED_TYPE" # Check that filtering and sorting is prettily disabled with pytest.raises(InvalidQueryError) as exc_info: diff --git a/tests/integration/test_webhook.py b/tests/integration/test_webhook.py index e281f5ea2..84bd95ade 100644 --- a/tests/integration/test_webhook.py +++ b/tests/integration/test_webhook.py @@ -24,15 +24,17 @@ def test_webhook_create_update(project, rand_gen): with pytest.raises(ValueError) as exc_info: webhook.update(status="invalid..") + valid_webhook_statuses = {item.value for item in Webhook.Status} assert str(exc_info.value) == \ - f"Value `invalid..` does not exist in supported values. Expected one of {[x.value for x in Webhook.WebhookStatus]}" + f"Value `invalid..` does not exist in supported values. Expected one of {valid_webhook_statuses}" with pytest.raises(ValueError) as exc_info: webhook.update(topics=["invalid.."]) + valid_webhook_topics = {item.value for item in Webhook.Topic} assert str(exc_info.value) == \ - f"Value `invalid..` does not exist in supported values. Expected one of {[x.value for x in Webhook.WebhookTopic]}" + f"Value `invalid..` does not exist in supported values. Expected one of {valid_webhook_topics}" with pytest.raises(TypeError) as exc_info: webhook.update(topics="invalid..") assert str(exc_info.value) == \ - "Topics must be List[Webhook.WebhookTopic]. Found `invalid..`" + "Topics must be List[Webhook.Topic]. Found `invalid..`" From b675332092e5d59e4133f3b1c8641bd80dde43d2 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Thu, 11 Mar 2021 11:09:23 -0500 Subject: [PATCH 12/13] requested changes --- labelbox/schema/dataset.py | 10 ++++++---- labelbox/schema/project.py | 6 +++--- tests/integration/test_labeling_parameter_overrides.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index c2b944c16..84cd5a379 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -1,12 +1,14 @@ +import os import json import logging +from itertools import islice from multiprocessing.dummy import Pool as ThreadPool -import os from labelbox.exceptions import InvalidQueryError, ResourceNotFoundError, InvalidAttributeError from labelbox.orm.db_object import DbObject, Updateable, Deletable from labelbox.orm.model import Entity, Field, Relationship +logger = logging.getLogger(__name__) class Dataset(DbObject, Updateable, Deletable): """ A Dataset is a collection of DataRows. @@ -185,7 +187,7 @@ def data_rows_for_external_id(self, external_id, limit=10): data_rows = self.data_rows(where=where) # Get at most `limit` data_rows. - data_rows = [row for row, _ in zip(data_rows, range(limit))] + data_rows = list(islice(data_rows, limit)) if not len(data_rows): raise ResourceNotFoundError(DataRow, where) @@ -209,7 +211,7 @@ def data_row_for_external_id(self, external_id): data_rows = self.data_rows_for_external_id(external_id=external_id, limit=2) if len(data_rows) > 1: - logging.warn( - f"More than one data_row has the provided external_id : `{external_id}`. Use function data_rows_for_external_id to fetch all" + logger.warning( + f"More than one data_row has the provided external_id : `%s`. Use function data_rows_for_external_id to fetch all", external_id ) return data_rows[0] diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index ade8f3ac0..a40cc6f7e 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -200,7 +200,7 @@ def upsert_instructions(self, instructions_file: str): frontendId = frontend.uid if frontend.name != "Editor": - logger.warn( + logger.warning( f"This function has only been tested to work with the Editor front end. Found %s", frontend.name) @@ -325,7 +325,7 @@ def validate_labeling_parameter_overrides(self, data): data_row, priority, num_labels = row if not isinstance(data_row, DataRow): raise TypeError( - f"Datarow should be be of type DataRow. Found {type(data_row)}. Index: {idx}" + f"data_row should be be of type DataRow. Found {type(data_row)}. Index: {idx}" ) for name, value in [["Priority", priority], @@ -358,7 +358,7 @@ def set_labeling_parameter_overrides(self, data): - Minimum priority is 1. * Priority is not the queue position. - The position is determined by the relative priority. - - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] + - E.g. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] will be assigned in the following order: [data_row_2, data_row_1, data_row_3] * Datarows with parameter overrides will appear before datarows without overrides. * The priority only effects items in the queue. diff --git a/tests/integration/test_labeling_parameter_overrides.py b/tests/integration/test_labeling_parameter_overrides.py index caaf773c4..5bd31d23f 100644 --- a/tests/integration/test_labeling_parameter_overrides.py +++ b/tests/integration/test_labeling_parameter_overrides.py @@ -50,7 +50,7 @@ def test_labeling_parameter_overrides(project, rand_gen): data = [(data_rows[12].uid, 1, 3)] project.set_labeling_parameter_overrides(data) assert str(exc_info.value) == \ - "Datarow should be be of type DataRow. Found . Index: 0" + "data_row should be be of type DataRow. Found . Index: 0" with pytest.raises(ValueError) as exc_info: data = [(data_rows[12], 0, 3)] From cc5b2cdb7d568f93a848507607954110ace527d4 Mon Sep 17 00:00:00 2001 From: Matt Sokoloff Date: Thu, 11 Mar 2021 11:10:20 -0500 Subject: [PATCH 13/13] yapf --- labelbox/schema/dataset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index 84cd5a379..b272b2380 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) + class Dataset(DbObject, Updateable, Deletable): """ A Dataset is a collection of DataRows. @@ -212,6 +213,6 @@ def data_row_for_external_id(self, external_id): limit=2) if len(data_rows) > 1: logger.warning( - f"More than one data_row has the provided external_id : `%s`. Use function data_rows_for_external_id to fetch all", external_id - ) + f"More than one data_row has the provided external_id : `%s`. Use function data_rows_for_external_id to fetch all", + external_id) return data_rows[0]