diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf3ff9b2..c4f5a4117 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +# Version 3.20.0 (2022-04-27) +## Added +* Batches in a project can be retrieved with `project.batches()` +* Added `Batch.remove_queued_data_rows()` to cancel remaining data rows in batch +* Added `Batch.export_data_rows()` which returns `DataRow`s for a batch + +## Updated +* NDJsonConverter now supports Video bounding box annotations. + * Note: Currently does not support nested classifications. + * Note: Converting an export into Labelbox annotation types, and back to export will result in only keyframe annotations. This is to support correct import format. + + +## Fix +* `batch.project()` now works + # Version 3.19.1 (2022-04-14) ## Fix * `create_data_rows` and `create_data_rows_sync` now uploads the file with a mimetype diff --git a/labelbox/__init__.py b/labelbox/__init__.py index a3636d46d..1788f718f 100644 --- a/labelbox/__init__.py +++ b/labelbox/__init__.py @@ -1,5 +1,12 @@ name = "labelbox" -__version__ = "3.19.1" +__version__ = "3.20.0" + +import sys +import warnings + +if sys.version_info < (3, 7): + warnings.warn("""Python 3.6 will no longer be actively supported + starting 06/01/2022. Please upgrade to Python 3.7 or higher.""") from labelbox.client import Client from labelbox.schema.project import Project diff --git a/labelbox/data/serialization/ndjson/label.py b/labelbox/data/serialization/ndjson/label.py index 195c6ed9a..529d711a7 100644 --- a/labelbox/data/serialization/ndjson/label.py +++ b/labelbox/data/serialization/ndjson/label.py @@ -2,10 +2,11 @@ from operator import itemgetter from typing import Dict, Generator, List, Tuple, Union from collections import defaultdict +import warnings from pydantic import BaseModel -from ...annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation, VideoClassificationAnnotation +from ...annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation, VideoClassificationAnnotation, VideoObjectAnnotation from ...annotation_types.collection import LabelCollection, LabelGenerator from ...annotation_types.data import ImageData, TextData, VideoData from ...annotation_types.label import Label @@ -15,12 +16,13 @@ from .metric import NDScalarMetric, NDMetricAnnotation, NDConfusionMatrixMetric from .classification import NDChecklistSubclass, NDClassification, NDClassificationType, NDRadioSubclass -from .objects import NDObject, NDObjectType +from .objects import NDObject, NDObjectType, NDSegments class NDLabel(BaseModel): annotations: List[Union[NDObjectType, NDClassificationType, - NDConfusionMatrixMetric, NDScalarMetric]] + NDConfusionMatrixMetric, NDScalarMetric, + NDSegments]] def to_common(self) -> LabelGenerator: grouped_annotations = defaultdict(list) @@ -37,15 +39,20 @@ def from_common(cls, yield from cls._create_video_annotations(label) def _generate_annotations( - self, grouped_annotations: Dict[str, List[Union[NDObjectType, - NDClassificationType, - NDConfusionMatrixMetric, - NDScalarMetric]]] + self, + grouped_annotations: Dict[str, + List[Union[NDObjectType, NDClassificationType, + NDConfusionMatrixMetric, + NDScalarMetric, NDSegments]]] ) -> Generator[Label, None, None]: for data_row_id, annotations in grouped_annotations.items(): annots = [] for annotation in annotations: - if isinstance(annotation, NDObjectType.__args__): + if isinstance(annotation, NDSegments): + annots.extend( + NDSegments.to_common(annotation, annotation.schema_id)) + + elif isinstance(annotation, NDObjectType.__args__): annots.append(NDObject.to_common(annotation)) elif isinstance(annotation, NDClassificationType.__args__): annots.extend(NDClassification.to_common(annotation)) @@ -55,7 +62,6 @@ def _generate_annotations( else: raise TypeError( f"Unsupported annotation. {type(annotation)}") - data = self._infer_media_type(annotations)(uid=data_row_id) yield Label(annotations=annots, data=data) @@ -65,7 +71,7 @@ def _infer_media_type( types = {type(annotation) for annotation in annotations} if TextEntity in types: return TextData - elif VideoClassificationAnnotation in types: + elif VideoClassificationAnnotation in types or VideoObjectAnnotation in types: return VideoData else: return ImageData @@ -83,26 +89,46 @@ def _get_consecutive_frames( def _create_video_annotations( cls, label: Label ) -> Generator[Union[NDChecklistSubclass, NDRadioSubclass], None, None]: + video_annotations = defaultdict(list) for annot in label.annotations: - if isinstance(annot, VideoClassificationAnnotation): + if isinstance( + annot, + (VideoClassificationAnnotation, VideoObjectAnnotation)): video_annotations[annot.feature_schema_id].append(annot) for annotation_group in video_annotations.values(): consecutive_frames = cls._get_consecutive_frames( sorted([annotation.frame for annotation in annotation_group])) - annotation = annotation_group[0] - frames_data = [] - for frames in consecutive_frames: - frames_data.append({'start': frames[0], 'end': frames[-1]}) - annotation.extra.update({'frames': frames_data}) - yield NDClassification.from_common(annotation, label.data) + + if isinstance(annotation_group[0], VideoClassificationAnnotation): + annotation = annotation_group[0] + frames_data = [] + for frames in consecutive_frames: + frames_data.append({'start': frames[0], 'end': frames[-1]}) + annotation.extra.update({'frames': frames_data}) + yield NDClassification.from_common(annotation, label.data) + + elif isinstance(annotation_group[0], VideoObjectAnnotation): + warnings.warn( + """Nested classifications are not currently supported + for video object annotations + and will not import alongside the object annotations.""") + segments = [] + for start_frame, end_frame in consecutive_frames: + segment = [] + for annotation in annotation_group: + if annotation.keyframe and start_frame <= annotation.frame <= end_frame: + segment.append(annotation) + segments.append(segment) + yield NDObject.from_common(segments, label.data) @classmethod def _create_non_video_annotations(cls, label: Label): non_video_annotations = [ annot for annot in label.annotations - if not isinstance(annot, VideoClassificationAnnotation) + if not isinstance(annot, (VideoClassificationAnnotation, + VideoObjectAnnotation)) ] for annotation in non_video_annotations: if isinstance(annotation, ClassificationAnnotation): diff --git a/labelbox/data/serialization/ndjson/objects.py b/labelbox/data/serialization/ndjson/objects.py index c23ad3815..ccf1e69f8 100644 --- a/labelbox/data/serialization/ndjson/objects.py +++ b/labelbox/data/serialization/ndjson/objects.py @@ -7,11 +7,13 @@ from pydantic import BaseModel from PIL import Image +from labelbox.data.annotation_types.data.video import VideoData + from ...annotation_types.data import ImageData, TextData, MaskData from ...annotation_types.ner import TextEntity from ...annotation_types.types import Cuid from ...annotation_types.geometry import Rectangle, Polygon, Line, Point, Mask -from ...annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation +from ...annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation, VideoObjectAnnotation from .classification import NDSubclassification, NDSubclassificationType from .base import DataRow, NDAnnotation @@ -20,6 +22,11 @@ class NDBaseObject(NDAnnotation): classifications: List[NDSubclassificationType] = [] +class VideoSupported(BaseModel): + #support for video for objects are per-frame basis + frame: int + + class _Point(BaseModel): x: float y: float @@ -118,6 +125,75 @@ def from_common(cls, rectangle: Rectangle, classifications=classifications) +class NDFrameRectangle(VideoSupported): + bbox: Bbox + + def to_common(self, feature_schema_id: Cuid) -> VideoObjectAnnotation: + return VideoObjectAnnotation( + frame=self.frame, + keyframe=True, + feature_schema_id=feature_schema_id, + value=Rectangle(start=Point(x=self.bbox.left, y=self.bbox.top), + end=Point(x=self.bbox.left + self.bbox.width, + y=self.bbox.top + self.bbox.height))) + + @classmethod + def from_common(cls, frame: int, rectangle: Rectangle): + return cls(frame=frame, + bbox=Bbox(top=rectangle.start.y, + left=rectangle.start.x, + height=rectangle.end.y - rectangle.start.y, + width=rectangle.end.x - rectangle.start.x)) + + +class NDSegment(BaseModel): + keyframes: List[NDFrameRectangle] + + @staticmethod + def lookup_segment_object_type(segment: List) -> "NDFrameObjectType": + """Used for determining which object type the annotation contains + returns the object type""" + result = {Rectangle: NDFrameRectangle}.get(type(segment[0].value)) + return result + + def to_common(self, feature_schema_id: Cuid): + return [ + keyframe.to_common(feature_schema_id) for keyframe in self.keyframes + ] + + @classmethod + def from_common(cls, segment): + nd_frame_object_type = cls.lookup_segment_object_type(segment) + + return cls(keyframes=[ + nd_frame_object_type.from_common(object_annotation.frame, + object_annotation.value) + for object_annotation in segment + ]) + + +class NDSegments(NDBaseObject): + segments: List[NDSegment] + + def to_common(self, feature_schema_id: Cuid): + result = [] + for segment in self.segments: + result.extend(NDSegment.to_common(segment, feature_schema_id)) + return result + + @classmethod + def from_common(cls, segments: List[VideoObjectAnnotation], data: VideoData, + feature_schema_id: Cuid, extra: Dict[str, + Any]) -> "NDSegments": + + segments = [NDSegment.from_common(segment) for segment in segments] + + return cls(segments=segments, + dataRow=DataRow(id=data.uid), + schema_id=feature_schema_id, + uuid=extra.get('uuid')) + + class _URIMask(BaseModel): instanceURI: str colorRGB: Tuple[int, int, int] @@ -208,9 +284,20 @@ def to_common(annotation: "NDObjectType") -> ObjectAnnotation: @classmethod def from_common( - cls, annotation: ObjectAnnotation, data: Union[ImageData, TextData] + cls, annotation: Union[ObjectAnnotation, + List[List[VideoObjectAnnotation]]], + data: Union[ImageData, TextData] ) -> Union[NDLine, NDPoint, NDPolygon, NDRectangle, NDMask, NDTextEntity]: obj = cls.lookup_object(annotation) + + #if it is video segments + if (obj == NDSegments): + return obj.from_common( + annotation, + data, + feature_schema_id=annotation[0][0].feature_schema_id, + extra=annotation[0][0].extra) + subclasses = [ NDSubclassification.from_common(annot) for annot in annotation.classifications @@ -220,15 +307,19 @@ def from_common( data) @staticmethod - def lookup_object(annotation: ObjectAnnotation) -> "NDObjectType": - result = { - Line: NDLine, - Point: NDPoint, - Polygon: NDPolygon, - Rectangle: NDRectangle, - Mask: NDMask, - TextEntity: NDTextEntity - }.get(type(annotation.value)) + def lookup_object( + annotation: Union[ObjectAnnotation, List]) -> "NDObjectType": + if isinstance(annotation, list): + result = NDSegments + else: + result = { + Line: NDLine, + Point: NDPoint, + Polygon: NDPolygon, + Rectangle: NDRectangle, + Mask: NDMask, + TextEntity: NDTextEntity + }.get(type(annotation.value)) if result is None: raise TypeError( f"Unable to convert object to MAL format. `{type(annotation.value)}`" @@ -238,3 +329,5 @@ def lookup_object(annotation: ObjectAnnotation) -> "NDObjectType": NDObjectType = Union[NDLine, NDPolygon, NDPoint, NDRectangle, NDMask, NDTextEntity] + +NDFrameObjectType = NDFrameRectangle diff --git a/labelbox/orm/model.py b/labelbox/orm/model.py index aaba9879b..e012a2b53 100644 --- a/labelbox/orm/model.py +++ b/labelbox/orm/model.py @@ -347,6 +347,7 @@ class Entity(metaclass=EntityMeta): Invite: Type[labelbox.Invite] InviteLimit: Type[labelbox.InviteLimit] ProjectRole: Type[labelbox.ProjectRole] + Project: Type[labelbox.Project] Batch: Type[labelbox.Batch] @classmethod diff --git a/labelbox/pagination.py b/labelbox/pagination.py index 6bec8fd0e..8d46901e5 100644 --- a/labelbox/pagination.py +++ b/labelbox/pagination.py @@ -25,7 +25,7 @@ def __init__(self, params: Dict[str, str], dereferencing: Union[List[str], Dict[str, Any]], obj_class: Union[Type["DbObject"], Callable[[Any, Any], Any]], - cursor_path: Optional[Dict[str, Any]] = None, + cursor_path: Optional[List[str]] = None, experimental: bool = False): """ Creates a PaginatedCollection. @@ -105,7 +105,7 @@ def get_next_page(self) -> Tuple[Dict[str, Any], bool]: class _CursorPagination(_Pagination): - def __init__(self, cursor_path: Dict[str, Any], *args, **kwargs): + def __init__(self, cursor_path: List[str], *args, **kwargs): super().__init__(*args, **kwargs) self.cursor_path = cursor_path self.next_cursor: Optional[Any] = None diff --git a/labelbox/schema/batch.py b/labelbox/schema/batch.py index 19b15b63f..53e4c7118 100644 --- a/labelbox/schema/batch.py +++ b/labelbox/schema/batch.py @@ -1,5 +1,18 @@ -from labelbox.orm.db_object import DbObject -from labelbox.orm.model import Field, Relationship +from typing import Generator, TYPE_CHECKING +from labelbox.orm.db_object import DbObject, experimental +from labelbox.orm import query +from labelbox.orm.model import Entity, Field, Relationship +from labelbox.exceptions import LabelboxError, ResourceNotFoundError +from io import StringIO +import ndjson +import requests +import logging +import time + +if TYPE_CHECKING: + from labelbox import Project + +logger = logging.getLogger(__name__) class Batch(DbObject): @@ -21,5 +34,87 @@ class Batch(DbObject): size = Field.Int("size") # Relationships - project = Relationship.ToOne("Project") created_by = Relationship.ToOne("User") + + def __init__(self, client, project_id, *args, **kwargs): + super().__init__(client, *args, **kwargs) + self.project_id = project_id + + def project(self) -> 'Project': # type: ignore + """ Returns Project which this Batch belongs to + + Raises: + LabelboxError: if the project is not found + """ + query_str = """query getProjectPyApi($projectId: ID!) { + project( + where: {id: $projectId}){ + %s + }}""" % query.results_query_part(Entity.Project) + params = {"projectId": self.project_id} + response = self.client.execute(query_str, params) + + if response is None: + raise ResourceNotFoundError(Entity.Project, params) + + return Entity.Project(self.client, response["project"]) + + def remove_queued_data_rows(self) -> None: + """ Removes remaining queued data rows from the batch and labeling queue. + + Args: + batch (Batch): Batch to remove queued data rows from + """ + + project_id_param = "projectId" + batch_id_param = "batchId" + self.client.execute("""mutation ArchiveBatchPyApi($%s: ID!, $%s: ID!) { + project(where: {id: $%s}) { archiveBatch(batchId: $%s) { id archivedAt } } + }""" % (project_id_param, batch_id_param, project_id_param, + batch_id_param), { + project_id_param: self.project_id, + batch_id_param: self.uid + }, + experimental=True) + + def export_data_rows(self, timeout_seconds=120) -> Generator: + """ Returns a generator that produces all data rows that are currently + in this batch. + + Note: For efficiency, the data are cached for 30 minutes. Newly created data rows will not appear + until the end of the cache period. + + Args: + timeout_seconds (float): Max waiting time, in seconds. + Returns: + Generator that yields DataRow objects belonging to this batch. + Raises: + LabelboxError: if the export fails or is unable to download within the specified time. + """ + id_param = "batchId" + query_str = """mutation GetBatchDataRowsExportUrlPyApi($%s: ID!) + {exportBatchDataRows(data:{batchId: $%s }) {downloadUrl createdAt status}} + """ % (id_param, id_param) + sleep_time = 2 + while True: + res = self.client.execute(query_str, {id_param: self.uid}) + res = res["exportBatchDataRows"] + if res["status"] == "COMPLETE": + download_url = res["downloadUrl"] + response = requests.get(download_url) + response.raise_for_status() + reader = ndjson.reader(StringIO(response.text)) + return ( + Entity.DataRow(self.client, result) for result in reader) + elif res["status"] == "FAILED": + raise LabelboxError("Data row export failed.") + + timeout_seconds -= sleep_time + if timeout_seconds <= 0: + raise LabelboxError( + f"Unable to export data rows within {timeout_seconds} seconds." + ) + + logger.debug("Batch '%s' data row export, waiting for server...", + self.uid) + time.sleep(sleep_time) diff --git a/labelbox/schema/dataset.py b/labelbox/schema/dataset.py index 6fe8ec86e..0a5d45251 100644 --- a/labelbox/schema/dataset.py +++ b/labelbox/schema/dataset.py @@ -314,9 +314,7 @@ def convert_item(item): items = [future.result() for future in as_completed(futures)] # Prepare and upload the desciptor file data = json.dumps(items) - return self.client.upload_data(data, - content_type="application/json", - filename="json_import.json") + return self.client.upload_data(data) def data_rows_for_external_id(self, external_id, diff --git a/labelbox/schema/organization.py b/labelbox/schema/organization.py index 4ebc126ac..4a5640f62 100644 --- a/labelbox/schema/organization.py +++ b/labelbox/schema/organization.py @@ -1,5 +1,5 @@ import json -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, List, Optional, Dict from labelbox.exceptions import LabelboxError from labelbox import utils @@ -129,7 +129,7 @@ def remove_user(self, user: "User") -> None: updateUser(where: {id: $%s}, data: {deleted: true}) { id deleted } }""" % (user_id_param, user_id_param), {user_id_param: user.uid}) - def create_resource_tag(self, tag=None) -> ResourceTag: + def create_resource_tag(self, tag: Dict[str, str]) -> ResourceTag: """ Creates a resource tag. >>> tag = {'text': 'tag-1', 'color': 'ffffff'} @@ -148,9 +148,13 @@ def create_resource_tag(self, tag=None) -> ResourceTag: query.results_query_part(ResourceTag)) params = { - tag_text_param: tag.get("text", ""), - tag_color_param: tag.get("color", "") + tag_text_param: tag.get("text", None), + tag_color_param: tag.get("color", None) } + if not all(params.values()): + raise ValueError( + f"tag must contain 'text' and 'color' keys. received: {tag}") + res = self.client.execute(query_str, params) return ResourceTag(self.client, res['createResourceTag']) diff --git a/labelbox/schema/project.py b/labelbox/schema/project.py index bc7059375..31b2af319 100644 --- a/labelbox/schema/project.py +++ b/labelbox/schema/project.py @@ -298,7 +298,7 @@ def export_labels(self, def _string_from_dict(dictionary: dict, value_with_quotes=False) -> str: """Returns a concatenated string of the dictionary's keys and values - + The string will be formatted as {key}: 'value' for each key. Value will be inclusive of quotations while key will not. This can be toggled with `value_with_quotes`""" @@ -609,7 +609,7 @@ def create_batch(self, name: str, data_rows: List[str], priority: int = 5): experimental=True)["project"][method] res['size'] = len(dr_ids) - return Entity.Batch(self.client, res) + return Entity.Batch(self.client, self.uid, res) def _update_queue_mode(self, mode: "Project.QueueMode") -> "Project.QueueMode": @@ -840,6 +840,24 @@ def bulk_import_requests(self) -> PaginatedCollection: ["bulkImportRequests"], Entity.BulkImportRequest) + def batches(self) -> PaginatedCollection: + """ Fetch all batches that belong to this project + + Returns: + A `PaginatedCollection of `Batch`es + """ + id_param = "projectId" + query_str = """query GetProjectBatchesPyApi($from: String, $first: PageSize, $%s: ID!) { + project(where: {id: $%s}) {id + batches(after: $from, first: $first) { nodes { %s } pageInfo { endCursor }}}} + """ % (id_param, id_param, query.results_query_part(Entity.Batch)) + return PaginatedCollection( + self.client, + query_str, {id_param: self.uid}, ['project', 'batches', 'nodes'], + lambda client, res: Entity.Batch(client, self.uid, res), + cursor_path=['project', 'batches', 'pageInfo', 'endCursor'], + experimental=True) + def upload_annotations( self, name: str, diff --git a/tests/data/serialization/ndjson/test_export_video_objects.py b/tests/data/serialization/ndjson/test_export_video_objects.py new file mode 100644 index 000000000..a10fef1be --- /dev/null +++ b/tests/data/serialization/ndjson/test_export_video_objects.py @@ -0,0 +1,605 @@ +from labelbox.data.annotation_types import Label +from labelbox.data.serialization.ndjson.converter import NDJsonConverter +from labelbox.data.annotation_types.geometry import Rectangle, Point +from labelbox.data.annotation_types import VideoData, VideoObjectAnnotation + + +def video_bbox_label(): + return Label( + uid='cl1z52xwh00050fhcmfgczqvn', + data=VideoData( + uid="cklr9mr4m5iao0rb6cvxu4qbn", + file_path=None, + frames=None, + url= + "https://storage.labelbox.com/ckcz6bubudyfi0855o1dt1g9s%2F26403a22-604a-a38c-eeff-c2ed481fb40a-cat.mp4?Expires=1651677421050&KeyName=labelbox-assets-key-3&Signature=vF7gMyfHzgZdfbB8BHgd88Ws-Ms" + ), + annotations=[ + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=46.0), + end=Point(extra={}, + x=454.0, + y=295.0)), + classifications=[], + frame=1, + keyframe=True), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=42.5), + end=Point(extra={}, + x=427.25, + y=308.25)), + classifications=[], + frame=2, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=39.0), + end=Point(extra={}, + x=400.5, + y=321.5)), + classifications=[], + frame=3, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=35.5), + end=Point(extra={}, + x=373.75, + y=334.75)), + classifications=[], + frame=4, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=32.0), + end=Point(extra={}, + x=347.0, + y=348.0)), + classifications=[], + frame=5, + keyframe=True), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=132.0), + end=Point(extra={}, + x=283.0, + y=348.0)), + classifications=[], + frame=9, + keyframe=True), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=122.333), + end=Point(extra={}, + x=295.5, + y=348.0)), + classifications=[], + frame=10, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=112.667), + end=Point(extra={}, + x=308.0, + y=348.0)), + classifications=[], + frame=11, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=103.0), + end=Point(extra={}, + x=320.5, + y=348.0)), + classifications=[], + frame=12, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=93.333), + end=Point(extra={}, + x=333.0, + y=348.0)), + classifications=[], + frame=13, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=83.667), + end=Point(extra={}, + x=345.5, + y=348.0)), + classifications=[], + frame=14, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=74.0), + end=Point(extra={}, + x=358.0, + y=348.0)), + classifications=[], + frame=15, + keyframe=True), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=66.833), + end=Point(extra={}, + x=387.333, + y=348.0)), + classifications=[], + frame=16, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=59.667), + end=Point(extra={}, + x=416.667, + y=348.0)), + classifications=[], + frame=17, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=52.5), + end=Point(extra={}, + x=446.0, + y=348.0)), + classifications=[], + frame=18, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=45.333), + end=Point(extra={}, + x=475.333, + y=348.0)), + classifications=[], + frame=19, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=38.167), + end=Point(extra={}, + x=504.667, + y=348.0)), + classifications=[], + frame=20, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=31.0), + end=Point(extra={}, + x=534.0, + y=348.0)), + classifications=[], + frame=21, + keyframe=True), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=29.5), + end=Point(extra={}, + x=543.0, + y=348.0)), + classifications=[], + frame=22, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=28.0), + end=Point(extra={}, + x=552.0, + y=348.0)), + classifications=[], + frame=23, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=26.5), + end=Point(extra={}, + x=561.0, + y=348.0)), + classifications=[], + frame=24, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=25.0), + end=Point(extra={}, + x=570.0, + y=348.0)), + classifications=[], + frame=25, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=23.5), + end=Point(extra={}, + x=579.0, + y=348.0)), + classifications=[], + frame=26, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=22.0), + end=Point(extra={}, + x=588.0, + y=348.0)), + classifications=[], + frame=27, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=20.5), + end=Point(extra={}, + x=597.0, + y=348.0)), + classifications=[], + frame=28, + keyframe=False), + VideoObjectAnnotation(name='bbox toy', + feature_schema_id='ckz38ofop0mci0z9i9w3aa9o4', + extra={ + 'value': 'bbox_toy', + 'instanceURI': None, + 'color': '#1CE6FF', + 'feature_id': 'cl1z52xw700000fhcayaqy0ev' + }, + value=Rectangle(extra={}, + start=Point(extra={}, + x=70.0, + y=19.0), + end=Point(extra={}, + x=606.0, + y=348.0)), + classifications=[], + frame=29, + keyframe=True) + ], + extra={ + 'Created By': + 'jtso@labelbox.com', + 'Project Name': + 'Pictor Video', + 'Created At': + '2022-04-14T15:11:19.000Z', + 'Updated At': + '2022-04-14T15:11:21.064Z', + 'Seconds to Label': + 0.0, + 'Agreement': + -1.0, + 'Benchmark Agreement': + -1.0, + 'Benchmark ID': + None, + 'Dataset Name': + 'cat', + 'Reviews': [], + 'View Label': + 'https://editor.labelbox.com?project=ckz38nsfd0lzq109bhq73est1&label=cl1z52xwh00050fhcmfgczqvn', + 'Has Open Issues': + 0.0, + 'Skipped': + False, + 'media_type': + 'video', + 'Data Split': + None + }) + + +def video_serialized_bbox_label(): + return { + 'uuid': + 'b24e672b-8f79-4d96-bf5e-b552ca0820d5', + 'dataRow': { + 'id': 'cklr9mr4m5iao0rb6cvxu4qbn' + }, + 'schemaId': + 'ckz38ofop0mci0z9i9w3aa9o4', + 'classifications': [], + 'segments': [{ + 'keyframes': [{ + 'frame': 1, + 'bbox': { + 'top': 46.0, + 'left': 70.0, + 'height': 249.0, + 'width': 384.0 + } + }, { + 'frame': 5, + 'bbox': { + 'top': 32.0, + 'left': 70.0, + 'height': 316.0, + 'width': 277.0 + } + }] + }, { + 'keyframes': [{ + 'frame': 9, + 'bbox': { + 'top': 132.0, + 'left': 70.0, + 'height': 216.0, + 'width': 213.0 + } + }, { + 'frame': 15, + 'bbox': { + 'top': 74.0, + 'left': 70.0, + 'height': 274.0, + 'width': 288.0 + } + }, { + 'frame': 21, + 'bbox': { + 'top': 31.0, + 'left': 70.0, + 'height': 317.0, + 'width': 464.0 + } + }, { + 'frame': 29, + 'bbox': { + 'top': 19.0, + 'left': 70.0, + 'height': 329.0, + 'width': 536.0 + } + }] + }] + } + + +def test_serialize_video_objects(): + label = video_bbox_label() + serialized_labels = NDJsonConverter.serialize([label]) + label = next(serialized_labels) + + manual_label = video_serialized_bbox_label() + + for key in label.keys(): + #ignore uuid because we randomize if there was none + if key != "uuid": + assert label[key] == manual_label[key] + + assert len(label['segments']) == 2 + assert len(label['segments'][0]['keyframes']) == 2 + assert len(label['segments'][1]['keyframes']) == 4 + + # #converts back only the keyframes. should be the sum of all prev segments + deserialized_labels = NDJsonConverter.deserialize([label]) + label = next(deserialized_labels) + assert len(label.annotations) == 6 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d4081a3db..72856ddc7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -51,8 +51,7 @@ def graphql_url(environ: str) -> str: hostname = os.environ.get('LABELBOX_TEST_ONPREM_HOSTNAME', None) if hostname is None: raise Exception(f"Missing LABELBOX_TEST_ONPREM_INSTANCE") - instance_id = hostname.split("-")[1].split(".")[0] - return f"https://app.replicated-{instance_id}.labelbox.dev/api/_gql" + return f"{hostname}/api/_gql" return 'http://host.docker.internal:8080/graphql' diff --git a/tests/integration/test_batch.py b/tests/integration/test_batch.py index 27b598d72..45ca27ac6 100644 --- a/tests/integration/test_batch.py +++ b/tests/integration/test_batch.py @@ -16,7 +16,19 @@ def big_dataset(dataset: Dataset): task.wait_till_done() yield dataset - dataset.delete() + + +@pytest.fixture +def small_dataset(dataset: Dataset): + task = dataset.create_data_rows([ + { + "row_data": IMAGE_URL, + "external_id": "my-image" + }, + ] * 3) + task.wait_till_done() + + yield dataset def test_create_batch(configured_project: Project, big_dataset: Dataset): @@ -26,3 +38,45 @@ def test_create_batch(configured_project: Project, big_dataset: Dataset): batch = configured_project.create_batch("test-batch", data_rows, 3) assert batch.name == 'test-batch' assert batch.size == len(data_rows) + + +def test_archive_batch(configured_project: Project, small_dataset: Dataset): + data_rows = [dr.uid for dr in list(small_dataset.export_data_rows())] + configured_project.update(queue_mode=Project.QueueMode.Batch) + batch = configured_project.create_batch("batch to archive", data_rows) + batch.remove_queued_data_rows() + exported_data_rows = list(batch.export_data_rows()) + + assert len(exported_data_rows) == 0 + + +def test_batch_project(configured_project: Project, small_dataset: Dataset): + data_rows = [dr.uid for dr in list(small_dataset.export_data_rows())] + configured_project.update(queue_mode=Project.QueueMode.Batch) + batch = configured_project.create_batch( + "batch to test project relationship", data_rows) + project_from_batch = batch.project() + + assert project_from_batch.uid == configured_project.uid + assert project_from_batch.name == configured_project.name + + +def test_export_data_rows(configured_project: Project, dataset: Dataset): + n_data_rows = 5 + task = dataset.create_data_rows([ + { + "row_data": IMAGE_URL, + "external_id": "my-image" + }, + ] * n_data_rows) + task.wait_till_done() + + data_rows = [dr.uid for dr in list(dataset.export_data_rows())] + configured_project.update(queue_mode=Project.QueueMode.Batch) + batch = configured_project.create_batch("batch test", data_rows) + + result = list(batch.export_data_rows()) + exported_data_rows = [dr.uid for dr in result] + + assert len(result) == n_data_rows + assert set(data_rows) == set(exported_data_rows) \ No newline at end of file diff --git a/tests/integration/test_data_row_metadata.py b/tests/integration/test_data_row_metadata.py index 9e9ec6404..8008b891a 100644 --- a/tests/integration/test_data_row_metadata.py +++ b/tests/integration/test_data_row_metadata.py @@ -46,7 +46,6 @@ def big_dataset(dataset: Dataset, image_url): task.wait_till_done() yield dataset - dataset.delete() def make_metadata(dr_id) -> DataRowMetadata: diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 9112a5298..2bfea262b 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -1,11 +1,10 @@ -import json import time import os import pytest import requests -from labelbox import Project, LabelingFrontend +from labelbox import Project, LabelingFrontend, Dataset from labelbox.exceptions import InvalidQueryError @@ -44,11 +43,19 @@ def test_project(client, rand_gen): assert set(final) == set(before) -@pytest.mark.skip( - reason="this will fail if run multiple times, limit is defaulted to 3 per org" - "add this back in when either all test orgs have unlimited, or we delete all tags befoer running" -) def test_update_project_resource_tags(client, rand_gen): + + def delete_tag(tag_id: str): + """Deletes a tag given the tag uid. Currently internal use only so this is not public""" + res = client.execute( + """mutation deleteResourceTagPyApi($tag_id: String!) { + deleteResourceTag(input: {id: $tag_id}) { + id + } + } + """, {"tag_id": tag_id}) + return res + before = list(client.get_projects()) for o in before: assert isinstance(o, Project) @@ -92,6 +99,9 @@ def test_update_project_resource_tags(client, rand_gen): assert len(project_resource_tag) == 1 assert project_resource_tag[0].uid == tagA.uid + delete_tag(tagA.uid) + delete_tag(tagB.uid) + def test_project_filtering(client, rand_gen): name_1 = rand_gen(str) @@ -191,3 +201,22 @@ def test_queue_mode(configured_project: Project): ) == configured_project.QueueMode.Dataset configured_project.update(queue_mode=configured_project.QueueMode.Batch) assert configured_project.queue_mode() == configured_project.QueueMode.Batch + + +def test_batches(configured_project: Project, dataset: Dataset, image_url): + task = dataset.create_data_rows([ + { + "row_data": image_url, + "external_id": "my-image" + }, + ] * 2) + task.wait_till_done() + configured_project.update(queue_mode=configured_project.QueueMode.Batch) + data_rows = [dr.uid for dr in list(dataset.export_data_rows())] + batch_one = 'batch one' + batch_two = 'batch two' + configured_project.create_batch(batch_one, [data_rows[0]]) + configured_project.create_batch(batch_two, [data_rows[1]]) + + names = set([batch.name for batch in list(configured_project.batches())]) + assert names == set([batch_one, batch_two]) \ No newline at end of file