From 7f9e44615b7b138d973c431ebc425d4336683a0d Mon Sep 17 00:00:00 2001 From: Paul Noirel Date: Tue, 27 Feb 2024 18:41:00 +0000 Subject: [PATCH 1/4] Update notebook for Custom metrics at the feature-level in Model --- .../custom_metrics_demo.ipynb | 1205 +++++++++++++---- 1 file changed, 917 insertions(+), 288 deletions(-) diff --git a/examples/model_diagnostics/custom_metrics_demo.ipynb b/examples/model_diagnostics/custom_metrics_demo.ipynb index a4c73a581..593395298 100644 --- a/examples/model_diagnostics/custom_metrics_demo.ipynb +++ b/examples/model_diagnostics/custom_metrics_demo.ipynb @@ -1,6 +1,6 @@ { "nbformat": 4, - "nbformat_minor": 5, + "nbformat_minor": 0, "metadata": {}, "cells": [ { @@ -15,28 +15,19 @@ { "metadata": {}, "source": [ + "\n", "\n", - "\n", "\n", "\n", "\n", - "\n", "" ], "cell_type": "markdown" }, - { - "metadata": {}, - "source": [ - " \n", - "\n", - " \n", - "" - ], - "cell_type": "markdown" - }, { "metadata": {}, "source": [ @@ -47,28 +38,21 @@ "* Measuring model quality is critical to efficiently building models. It is important that the metrics used to measure model quality closely align with the business objectives for the model. Otherwise, slight changes in model quality, as they related to these core objectives, are lost to noise. Custom metrics enables users to measure model quality in terms of their exact business goals. By incorporating custom metrics into workflows, users can:\n", " * Iterate faster\n", " * Measure and report on model quality\n", - " * Understand marginal value of additional labels and modeling efforts\n", - "\n", - " \n", - "* Checkout this [notebook](custom_metrics_basics.ipynb) for more details on the metrics types and built in functions for calculating metrics.\n" + " * Understand marginal value of additional labels and modeling efforts\n" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "## Environment Setup\n", - "\n", - "Install dependencies" + "## Environment setup" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "!pip install -q \"labelbox[data]\" \\\n", - " scikit-image \\\n", - " tensorflow" + "!pip install -q 'labelbox[data]'" ], "cell_type": "code", "outputs": [], @@ -77,13 +61,22 @@ { "metadata": {}, "source": [ - "# Run these if running in a colab notebook\n", - "COLAB = \"google.colab\" in str(get_ipython())\n", - "\n", - "if COLAB:\n", - " !git clone https://github.com/Labelbox/labelbox-python.git\n", - " !cd labelbox-python\n", - " !mv labelbox-python/examples/model_assisted_labeling/*.py ." + "from labelbox.schema.ontology import OntologyBuilder, Tool, Classification, Option\n", + "from labelbox import Client, MALPredictionImport, LabelImport\n", + "from labelbox.data.serialization import NDJsonConverter\n", + "from labelbox.schema.media_type import MediaType\n", + "from labelbox.schema.quality_mode import QualityMode\n", + "from labelbox.data.metrics.group import get_label_pairs\n", + "from labelbox.data.metrics import feature_miou_metric, feature_confusion_matrix_metric\n", + "from labelbox.data.annotation_types import (\n", + " Label, ImageData, ObjectAnnotation, MaskData,\n", + " Rectangle, Point, Line, Mask, Polygon,\n", + " Radio, Checklist, Text,\n", + " ClassificationAnnotation, ClassificationAnswer, RelationshipAnnotation, Relationship\n", + ")\n", + "import uuid\n", + "import numpy as np\n", + "from labelbox.schema.queue_mode import QueueMode" ], "cell_type": "code", "outputs": [], @@ -92,39 +85,17 @@ { "metadata": {}, "source": [ - "Import libraries" + "## Replace with your API Key\n", + "Guides on [Create an API key](https://docs.labelbox.com/docs/create-an-api-key)" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "import sys\n", - "\n", - "sys.path.append('../model_assisted_labeling')\n", + "API_KEY = \"\"\n", "\n", - "import uuid\n", - "import numpy as np\n", - "from skimage import measure\n", - "import requests\n", - "from tqdm import notebook\n", - "import requests\n", - "import csv\n", - "import os\n", - "\n", - "import labelbox as lb\n", - "import labelbox.types as lb_types\n", - "from labelbox.data.metrics.group import get_label_pairs\n", - "from labelbox.data.metrics import (feature_miou_metric, confusion_matrix_metric,\n", - " feature_confusion_matrix_metric)\n", - "\n", - "try:\n", - " from image_model import predict, load_model\n", - "except ModuleNotFoundError:\n", - " # !git clone https://github.com/Labelbox/labelbox-python.git\n", - " # !cd labelbox-python && git checkout mea-dev\n", - " # !mv labelbox-python/examples/model_assisted_labeling/*.py .\n", - " raise Exception(\"You will need to run from the labelbox-python git repo\")" + "client = Client(API_KEY)" ], "cell_type": "code", "outputs": [], @@ -133,16 +104,337 @@ { "metadata": {}, "source": [ - "Configure client" + "## Supported Predictions" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "PROJECT_NAME = \"Diagnostics Demo Custom Metrics\"\n", - "MODEL_NAME = \"MSCOCO Mapillary Custom Metrics\"\n", - "MODEL_VERSION = \"0.0.0\"" + "########### Radio Classification ###########\n", + "\n", + "# Python annotation\n", + "radio_prediction = ClassificationAnnotation(\n", + " name=\"radio_question\",\n", + " value=Radio(answer = ClassificationAnswer(name = \"first_radio_answer\", confidence=0.5))\n", + ")\n", + "\n", + "# NDJSON\n", + "radio_prediction_ndjson = {\n", + " 'name': 'radio_question',\n", + " 'answer': {'name': 'first_radio_answer', 'confidence': 0.5}\n", + "}\n", + "\n", + "########## Nested Classifications are only supported with NDJSON tools ##########\n", + "\n", + "nested_radio_prediction_ndjson = {\n", + " \"name\": \"nested_radio_question\",\n", + " \"confidence\": 0.5 ,\n", + " \"answer\": { \"name\": \"first_radio_answer\", \"confidence\": 0.5 },\n", + " \"classifications\" : [\n", + " {\n", + " \"name\": \"sub_radio_question\",\n", + " \"answer\": {\"name\": \"first_sub_radio_answer\", \"confidence\": 0.5 }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "\n", + "nested_checklist_prediction_ndjson = {\n", + " \"name\": \"nested_checklist_question\",\n", + " \"confidence\": 0.5 ,\n", + " \"answer\": [{\n", + " \"name\": \"first_checklist_answer\",\n", + " \"confidence\": 0.5,\n", + " \"classifications\" : [\n", + " {\n", + " \"name\": \"sub_checklist_question\",\n", + " \"answer\": {\"name\": \"first_sub_checklist_answer\", \"confidence\": 0.5 }\n", + " }\n", + " ]\n", + " }]\n", + "}\n", + "\n", + "############ Checklist ############\n", + "\n", + "# Python Annotations\n", + "checklist_prediction = ClassificationAnnotation(\n", + " name=\"checklist_question\", # must match your ontology feature's name\n", + " value=Checklist(\n", + " answer = [\n", + " ClassificationAnswer(\n", + " name = \"first_checklist_answer\",\n", + " confidence=0.5\n", + " ),\n", + " ClassificationAnswer(\n", + " name = \"second_checklist_answer\",\n", + " confidence=0.5\n", + " )\n", + " ]\n", + " )\n", + " )\n", + "\n", + "# NDJSON\n", + "checklist_prediction_ndjson = {\n", + " 'name': 'checklist_question',\n", + " 'answer': [\n", + " {'name': 'first_checklist_answer' , 'confidence': 0.5},\n", + " {'name': 'second_checklist_answer', 'confidence': 0.5}\n", + " ]\n", + "}\n", + "\n", + "####### Bounding box #######\n", + "\n", + "\n", + "# Python Annotation\n", + "bbox_prediction = ObjectAnnotation(\n", + " name = \"bounding_box\", # must match your ontology feature's name\n", + " confidence=0.5,\n", + " value=Rectangle(\n", + " start=Point(x=977, y=1690), # Top left\n", + " end=Point(x=330, y=225), # Bottom right\n", + " ),\n", + "\n", + ")\n", + "\n", + "#NDJSON\n", + "bbox_prediction_ndjson = {\n", + " 'name': 'bounding_box',\n", + " 'confidence': 0.5,\n", + " 'bbox': {\n", + " \"top\": 500,\n", + " \"left\": 1190,\n", + " \"height\": 230,\n", + " \"width\": 225\n", + " }\n", + "}\n", + "\n", + "####### Bounding box with nested classification #######\n", + "bbox_with_radio_subclass_prediction = ObjectAnnotation(\n", + " name=\"bbox_with_radio_subclass\",\n", + " confidence=0.5, # must match your ontology feature's name\n", + " value=Rectangle(\n", + " start=Point(x=433, y=341), # Top left\n", + " end=Point(x=291, y=230), # Bottom right\n", + " ),\n", + " classifications=[\n", + " \tClassificationAnnotation(\n", + " \tname=\"sub_radio_question\",\n", + " \t\tvalue=Radio(answer=ClassificationAnswer(name=\"first_sub_radio_answer\", confidence=0.5))\n", + " )\n", + " ]\n", + ")\n", + "\n", + "\n", + "## NDJSON\n", + "bbox_with_radio_subclass_prediction_ndjson = {\n", + " \"name\": \"bbox_with_radio_subclass\",\n", + " \"confidence\": 0.5,\n", + " \"classifications\": [{\n", + " \"name\": \"sub_radio_question\",\n", + " \"confidence\": 0.5,\n", + " \"answer\":\n", + " { \"name\":\"first_sub_radio_answer\", \"confidence\": 0.5}\n", + "\n", + " }],\n", + " \"bbox\": {\n", + " \"top\": 533,\n", + " \"left\": 641,\n", + " \"height\": 291,\n", + " \"width\": 330\n", + " }\n", + "}\n", + "\n", + "\n", + "####### Bounding box with nested free text #######\n", + "\n", + "## NDJSON\n", + "bbox_with_free_text_subclass_prediction_ndjson = {\n", + " \"name\": \"bbox_with_free_text_subclass\",\n", + " \"confidence\": 0.5,\n", + " \"classifications\": [{\n", + " \"name\": \"text\",\n", + " \"confidence\": 0.5,\n", + " 'answer': 'sample text',\n", + "\n", + " }],\n", + " \"bbox\": {\n", + " \"top\": 933,\n", + " \"left\": 541,\n", + " \"height\": 191,\n", + " \"width\": 330\n", + " }\n", + "}\n", + "\n", + "########## Polygon ##########\n", + "# Python Anotation\n", + "polygon_prediction = ObjectAnnotation(\n", + " name = \"polygon\", # must match your ontology feature's name\n", + " confidence = 0.5,\n", + " value=Polygon( # Coordinates for the verticies of your polygon\n", + " points=[Point(x=1489.581,y=183.934),Point(x=2278.306,y=256.885),Point(x=2428.197,y=200.437),Point(x=2560.0,y=335.419),\n", + " Point(x=2557.386,y=503.165),Point(x=2320.596,y=503.103),Point(x=2156.083, y=628.943),Point(x=2161.111,y=785.519),\n", + " Point(x=2002.115, y=894.647),Point(x=1838.456,y=877.874),Point(x=1436.53,y=874.636),Point(x=1411.403,y=758.579),\n", + " Point(x=1353.853,y=751.74),Point(x=1345.264, y=453.461),Point(x=1426.011,y=421.129)]\n", + " ),\n", + ")\n", + "\n", + "polygon_prediction_target = ObjectAnnotation(\n", + " name = \"polygon\", # must match your ontology feature's name\n", + " confidence = 0.5,\n", + " value=Polygon( # Coordinates for the verticies of your polygon\n", + " points=[Point(x=1089.581,y=203.934),Point(x=1878.306,y=276.885),Point(x=2028.197,y=220.437),Point(x=2160.0,y=355.419),\n", + " Point(x=2157.386,y=523.165),Point(x=2020.596,y=523.103),Point(x=1756.083, y=648.943),Point(x=1761.111,y=805.519),\n", + " Point(x=1602.115, y=914.647),Point(x=1438.456,y=897.874),Point(x=1036.53,y=894.636),Point(x=1011.403,y=778.579),\n", + " Point(x=953.853,y=771.74),Point(x=945.264, y=473.461),Point(x=926.011,y=441.129)]\n", + " ),\n", + ")\n", + "\n", + "polygon_relationship = RelationshipAnnotation(\n", + " name=\"relationship\",\n", + " value=Relationship(\n", + " source=polygon_prediction,\n", + " target=polygon_prediction_target,\n", + " type=Relationship.Type.UNIDIRECTIONAL,\n", + " ))\n", + "\n", + "\n", + "# NDJSON\n", + "\n", + "polygon_prediction_ndjson = {\n", + " 'name': 'polygon',\n", + " 'confidence': 0.5,\n", + " 'polygon': [\n", + " {'x': 1489.581, 'y': 183.934},\n", + " {'x': 2278.306, 'y': 256.885},\n", + " {'x': 2428.197, 'y': 200.437},\n", + " {'x': 2560.0, 'y': 335.419},\n", + " {'x': 2557.386, 'y': 503.165},\n", + " {'x': 2320.596, 'y': 503.103},\n", + " {'x': 2156.083, 'y': 628.943},\n", + " {'x': 2161.111, 'y': 785.519},\n", + " {'x': 2002.115, 'y': 894.647},\n", + " {'x': 1838.456, 'y': 877.874},\n", + " {'x': 1436.53, 'y': 874.636},\n", + " {'x': 1411.403, 'y': 758.579},\n", + " {'x': 1353.853, 'y': 751.74},\n", + " {'x': 1345.264, 'y': 453.461},\n", + " {'x': 1426.011, 'y': 421.129},\n", + " {'x': 1489.581, 'y': 183.934}\n", + " ]\n", + "}\n", + "\n", + "####### Free text #######\n", + "# Confidence is not supported for text prediction\n", + "# Python annotation\n", + "text_annotation = ClassificationAnnotation(\n", + " name=\"free_text\", # must match your ontology feature's name\n", + " value=Text(answer=\"sample text\")\n", + ")\n", + "\n", + "# NDJSON\n", + "text_annotation_ndjson = {\n", + " 'name': 'free_text',\n", + " 'answer': 'sample text',\n", + "}\n", + "\n", + "######### Segmentation mask #########\n", + "\n", + "# Python\n", + "# Identifying what values in the numpy array correspond to the mask annotation\n", + "color = (0, 0, 0)\n", + "\n", + "# convert a polygon to mask\n", + "im_height, im_width = 100,100 #need to provide the height and width of image.\n", + "mask_data = MaskData(arr=\n", + " polygon_prediction.value.draw(height=im_height,width=im_width,color=color))\n", + "\n", + "# convert a 2D array to 3D array\n", + "arr_2d = np.zeros((100,100), dtype='uint8')\n", + "mask_data = MaskData.from_2D_arr(arr_2d)\n", + "\n", + "# a 3D array where 3rd axis is RGB values.\n", + "mask_data = MaskData(arr= np.zeros([400,450,3],dtype='uint8'))\n", + "\n", + "mask_prediction = ObjectAnnotation(\n", + " name = \"mask\", # must match your ontology feature's name\n", + " confidence=0.5,\n", + " value=Mask(mask=mask_data, color=color),\n", + ")\n", + "\n", + "mask_prediction_ndjson = {\n", + " \"name\": \"mask\",\n", + " \"confidence\": 0.5,\n", + " \"classifications\": [],\n", + " \"mask\": {\"instanceURI\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png\",\n", + " \"colorRGB\": (255, 255, 255)}\n", + "}\n", + "\n", + "######## Point ########\n", + "\n", + "# Python Annotation\n", + "point_prediction = ObjectAnnotation(\n", + " name = \"point\", # must match your ontology feature's name\n", + " confidence=0.5,\n", + " value = Point(x=1166.606, y=1441.768),\n", + ")\n", + "\n", + "\n", + "# NDJSON\n", + "point_prediction_ndjson = {\n", + " 'name': 'point',\n", + " 'confidence': 0.5,\n", + " 'classifications': [],\n", + " 'point': {'x': 1166.606, 'y': 1441.768}\n", + "}\n", + "\n", + "###### Polyline ######\n", + "\n", + "\n", + "# Python Annotation\n", + "\n", + "polyline_prediction = ObjectAnnotation(\n", + " name = \"polyline\", # must match your ontology feature's name\n", + " confidence=0.5, ## Not supported for python annotation tools\n", + " value=Line( # Coordinates for the keypoints in your polyline\n", + " points=[Point(x=2534.353, y=249.471),Point(x=2429.492, y=182.092),Point(x=2294.322, y=221.962),Point(x=2224.491, y=180.463),Point(x=2136.123, y=204.716),\n", + " Point(x=1712.247, y=173.949),Point(x=1703.838, y=84.438),Point(x=1579.772, y=82.61),Point(x=1583.442, y=167.552),\n", + " Point(x=1478.869, y=164.903),Point(x=1418.941, y=318.149),Point(x=1243.128, y=400.815),Point(x=1022.067, y=319.007),\n", + " Point(x=892.367, y=379.216),Point(x=670.273, y=364.408),Point(x=613.114, y=288.16),Point(x=377.559, y=238.251),\n", + " Point(x=368.087, y=185.064),Point(x=246.557, y=167.286),Point(x=236.648, y=285.61),Point(x=90.929, y=326.412)]\n", + " ),\n", + ")\n", + "\n", + "# NDJSON\n", + "polyline_prediction_ndjson = {\n", + " 'name': 'polyline',\n", + " 'confidence':0.5,\n", + " 'classifications': [],\n", + " 'line': [\n", + " {'x': 2534.353, 'y': 249.471},\n", + " {'x': 2429.492, 'y': 182.092},\n", + " {'x': 2294.322, 'y': 221.962},\n", + " {'x': 2224.491, 'y': 180.463},\n", + " {'x': 2136.123, 'y': 204.716},\n", + " {'x': 1712.247, 'y': 173.949},\n", + " {'x': 1703.838, 'y': 84.438},\n", + " {'x': 1579.772, 'y': 82.61},\n", + " {'x': 1583.442, 'y': 167.552},\n", + " {'x': 1478.869, 'y': 164.903},\n", + " {'x': 1418.941, 'y': 318.149},\n", + " {'x': 1243.128, 'y': 400.815},\n", + " {'x': 1022.067, 'y': 319.007},\n", + " {'x': 892.367, 'y': 379.216},\n", + " {'x': 670.273, 'y': 364.408},\n", + " {'x': 613.114, 'y': 288.16},\n", + " {'x': 377.559, 'y': 238.251},\n", + " {'x': 368.087, 'y': 185.064},\n", + " {'x': 246.557, 'y': 167.286},\n", + " {'x': 236.648, 'y': 285.61},\n", + " {'x': 90.929, 'y': 326.412}\n", + " ]\n", + "}\n", + "\n" ], "cell_type": "code", "outputs": [], @@ -151,19 +443,23 @@ { "metadata": {}, "source": [ - "API Key\n", - "\n", - "Provide a valid api key below in order to properly connect to the Labelbox Client." + "## Step 1: Import data rows into Catalog" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "# Add your api key\n", - "API_KEY = None\n", - "client = lb.Client(api_key=API_KEY)\n", - "load_model() # initialize Tensorflow Model" + "NUM_DATA_ROWS = 1\n", + "\n", + "# send a sample image as batch to the project\n", + "test_img_urls = [{\n", + " \"row_data\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg\",\n", + " \"global_key\": str(uuid.uuid4())\n", + "} for i in range(0, NUM_DATA_ROWS)]\n", + "dataset = client.create_dataset(name=\"image_prediction_demo\", iam_integration=None)\n", + "data_rows = dataset.create_data_rows(test_img_urls)\n", + "print(data_rows)" ], "cell_type": "code", "outputs": [], @@ -172,64 +468,109 @@ { "metadata": {}, "source": [ - "# Configure for whatever combination of tools and class names that you would like.\n", - "class_mappings = {\n", - " 1: {\n", - " \"name\": 'person',\n", - " \"kind\": lb.Tool.Type.POLYGON\n", - " },\n", - " 2: {\n", - " \"name\": 'bicycle',\n", - " \"kind\": lb.Tool.Type.SEGMENTATION,\n", - " 'color': 64\n", - " },\n", - " 3: {\n", - " \"name\": 'car',\n", - " \"kind\": lb.Tool.Type.BBOX\n", - " },\n", - " 4: {\n", - " \"name\": 'motorcycle',\n", - " \"kind\": lb.Tool.Type.BBOX\n", - " },\n", - " 6: {\n", - " \"name\": 'bus',\n", - " \"kind\": lb.Tool.Type.POLYGON\n", - " },\n", - " 7: {\n", - " \"name\": 'train',\n", - " \"kind\": lb.Tool.Type.POLYGON\n", - " },\n", - " 8: {\n", - " \"name\": 'truck',\n", - " \"kind\": lb.Tool.Type.POLYGON\n", - " },\n", - " 10: {\n", - " \"name\": 'traffic light',\n", - " \"kind\": lb.Tool.Type.POINT\n", - " },\n", - " 11: {\n", - " \"name\": 'fire hydrant',\n", - " \"kind\": lb.Tool.Type.BBOX\n", - " },\n", - " 13: {\n", - " \"name\": 'stop sign',\n", - " \"kind\": lb.Tool.Type.SEGMENTATION,\n", - " 'color': 255\n", - " },\n", - " 14: {\n", - " \"name\": 'parking meter',\n", - " \"kind\": lb.Tool.Type.POINT\n", - " },\n", - " 28: {\n", - " \"name\": 'umbrella',\n", - " \"kind\": lb.Tool.Type.SEGMENTATION,\n", - " 'color': 128\n", - " },\n", - " 31: {\n", - " \"name\": 'handbag',\n", - " \"kind\": lb.Tool.Type.POINT\n", - " },\n", - "}" + "## Step 2: Create/select an Ontology for your model predictions\n", + "Your project should have the correct ontology setup with all the tools and classifications supported for your annotations, and the tool names and classification instructions should match the name/instructions fields in your annotations to ensure the correct feature schemas are matched.\n" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "ontology_builder = OntologyBuilder(\n", + " classifications=[ # List of Classification objects\n", + " Classification( # Radio classification given the name \"text\" with two options: \"first_radio_answer\" and \"second_radio_answer\"\n", + " class_type=Classification.Type.RADIO,\n", + " name=\"radio_question\",\n", + " options=[\n", + " Option(value=\"first_radio_answer\"),\n", + " Option(value=\"second_radio_answer\")\n", + " ]\n", + " ),\n", + " Classification( # Checklist classification given the name \"text\" with two options: \"first_checklist_answer\" and \"second_checklist_answer\"\n", + " class_type=Classification.Type.CHECKLIST,\n", + " name=\"checklist_question\",\n", + " options=[\n", + " Option(value=\"first_checklist_answer\"),\n", + " Option(value=\"second_checklist_answer\")\n", + " ]\n", + " ),\n", + " Classification( # Text classification given the name \"text\"\n", + " class_type=Classification.Type.TEXT,\n", + " name=\"free_text\"\n", + " ),\n", + " Classification(\n", + " class_type=Classification.Type.RADIO,\n", + " name=\"nested_radio_question\",\n", + " options=[\n", + " Option(\"first_radio_answer\",\n", + " options=[\n", + " Classification(\n", + " class_type=Classification.Type.RADIO,\n", + " name=\"sub_radio_question\",\n", + " options=[Option(\"first_sub_radio_answer\")]\n", + " )\n", + " ]\n", + " )\n", + " ]\n", + " ),\n", + " Classification(\n", + " class_type=Classification.Type.CHECKLIST,\n", + " name=\"nested_checklist_question\",\n", + " options=[\n", + " Option(\"first_checklist_answer\",\n", + " options=[\n", + " Classification(\n", + " class_type=Classification.Type.CHECKLIST,\n", + " name=\"sub_checklist_question\",\n", + " options=[Option(\"first_sub_checklist_answer\")]\n", + " )\n", + " ]\n", + " )\n", + " ]\n", + " ),\n", + " ],\n", + " tools=[ # List of Tool objects\n", + " Tool( # Bounding Box tool given the name \"box\"\n", + " tool=Tool.Type.BBOX,\n", + " name=\"bounding_box\"),\n", + " Tool( # Bounding Box tool given the name \"box\"\n", + " tool=Tool.Type.BBOX,\n", + " name=\"bbox_with_radio_subclass\",\n", + " classifications=[\n", + " Classification(\n", + " class_type=Classification.Type.RADIO,\n", + " name=\"sub_radio_question\",\n", + " options=[\n", + " Option(value=\"first_sub_radio_answer\")\n", + " ]\n", + " ),\n", + " ]\n", + " ),\n", + " Tool( # Bounding Box tool given the name \"box\"\n", + " tool=Tool.Type.BBOX,\n", + " name=\"bbox_with_free_text_subclassification\",\n", + " classifications=[\n", + " Classification(\n", + " class_type=Classification.Type.TEXT,\n", + " name=\"sub_text_classification\",\n", + " ),\n", + " ]\n", + " ),\n", + " Tool( # Polygon tool given the name \"polygon\"\n", + " tool=Tool.Type.POLYGON,\n", + " name=\"polygon\"),\n", + " Tool( # Segmentation mask tool given the name \"mask\"\n", + " tool=Tool.Type.RASTER_SEGMENTATION,\n", + " name=\"mask\"),\n", + " \t Tool( # Point tool given the name \"point\"\n", + " tool=Tool.Type.POINT,\n", + " name=\"point\"),\n", + " Tool( # Polyline tool given the name \"line\"\n", + " tool=Tool.Type.LINE,\n", + " name=\"polyline\")]\n", + ")\n", + "\n", + "ontology = client.create_ontology(\"Image Prediction Import Demo\", ontology_builder.asdict(), media_type=MediaType.Image)" ], "cell_type": "code", "outputs": [], @@ -238,22 +579,18 @@ { "metadata": {}, "source": [ - "## Create Predictions\n", - "* Loop over data_rows, make predictions, and create ndjson" + "## Step 3: Create a Model and Model Run" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "# --- setup dataset ---\n", - "# load mapillary sample\n", - "sample_csv_url = \"https://raw.githubusercontent.com/Labelbox/labelbox-python/master/examples/assets/mapillary_sample.csv\"\n", - "with requests.get(sample_csv_url, stream=True) as r:\n", - " image_data = [\n", - " row.split(',')\n", - " for row in (line.decode('utf-8') for line in r.iter_lines())\n", - " ]" + "# create Model\n", + "model = client.create_model(name=\"with_aggregated_custom_metrics\" + str(uuid.uuid4()),\n", + " ontology_id=ontology.uid)\n", + "# create Model Run\n", + "model_run = model.create_model_run(\"iteration 1\")" ], "cell_type": "code", "outputs": [], @@ -262,42 +599,14 @@ { "metadata": {}, "source": [ - "predictions = list()\n", - "for (image_url, external_id) in notebook.tqdm(image_data):\n", - " image = lb_types.ImageData(url=image_url, external_id=external_id)\n", - " height, width = image.value.shape[:2]\n", - " prediction = predict(np.array([image.im_bytes]),\n", - " min_score=0.5,\n", - " height=height,\n", - " width=width)\n", - " boxes, classes, seg_masks = prediction[\"boxes\"], prediction[\n", - " \"class_indices\"], prediction[\"seg_masks\"]\n", - " annotations = []\n", - " for box, class_idx, seg in zip(boxes, classes, seg_masks):\n", - " if class_idx in class_mappings:\n", - " class_info = class_mappings.get(class_idx)\n", - " if class_info['kind'] == lb.Tool.Type.POLYGON:\n", - " contours = measure.find_contours(seg, 0.5)\n", - " pts = contours[0].astype(np.int32)\n", - " value = lb_types.Polygon(points=[\n", - " lb_types.Point(x=x, y=y) for x, y in np.roll(pts, 1, axis=-1)\n", - " ])\n", - " elif class_info['kind'] == lb.Tool.Type.BBOX:\n", - " value = lb_types.Rectangle(start=lb_types.Point(x=box[1], y=box[0]),\n", - " end=lb_types.Point(x=box[3], y=box[2]))\n", - " elif class_info['kind'] == lb.Tool.Type.POINT:\n", - " value = lb_types.Point(x=(box[1] + box[3]) / 2.,\n", - " y=(box[0] + box[2]) / 2.)\n", - " elif class_info['kind'] == lb.Tool.Type.SEGMENTATION:\n", - " value = lb_types.Mask(mask=lb_types.MaskData.from_2D_arr(seg *\n", - " class_info['color']),\n", - " color=(class_info['color'],) * 3)\n", - " else:\n", - " raise ValueError(\n", - " f\"Unsupported kind found. {class_info['kind']}\")\n", - " annotations.append(\n", - " lb_types.ObjectAnnotation(name=class_info['name'], value=value))\n", - " predictions.append(lb_types.Label(data=image, annotations=annotations))" + "## Step 4: Send data rows to the Model Run" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "model_run.upsert_data_rows(global_keys=[data_row.global_key for data_row in dataset.export_data_rows()])" ], "cell_type": "code", "outputs": [], @@ -306,17 +615,34 @@ { "metadata": {}, "source": [ - "## Setup a project" + "## Step 5. Create the predictions payload\n", + "\n", + "Create the prediction payload using the snippets of code in ***Supported Predictions*** section.\n", + "\n", + "The resulting label_ndjson should have exactly the same content for predictions that are supported by both (with exception of the uuid strings that are generated)" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "tools = []\n", - "for target in class_mappings.values():\n", - " tools.append(lb.Tool(tool=target['kind'], name=target[\"name\"]))\n", - "ontology_builder = lb.OntologyBuilder(tools=tools)" + "# Create a Label for predictions\n", + "label_predictions = [Label(\n", + " data=ImageData(uid=data_row.uid),\n", + " annotations = [\n", + " radio_prediction,\n", + " checklist_prediction,\n", + " bbox_prediction,\n", + " bbox_with_radio_subclass_prediction,\n", + " polygon_prediction,\n", + " # mask_prediction,\n", + " point_prediction,\n", + " text_annotation,\n", + " ]\n", + ") for data_row in dataset.export_data_rows()]\n", + "\n", + "# Convert the prediction label from a Labelbox class object to the underlying NDJSON format required for upload - uploads can be directly built in this syntax as well\n", + "ndjson_prediction = list(NDJsonConverter.serialize(label_predictions))" ], "cell_type": "code", "outputs": [], @@ -325,39 +651,134 @@ { "metadata": {}, "source": [ - "print(f\"Setting up: {PROJECT_NAME}\")\n", - "\n", - "project = client.create_project(name=PROJECT_NAME, media_type=lb.MediaType.Image)\n", - "editor = next(\n", - " client.get_labeling_frontends(where=lb.LabelingFrontend.name == \"Editor\"))\n", - "project.setup(editor, ontology_builder.asdict())\n", - "\n", - "dataset = client.create_dataset(name=\"Mapillary Diagnostics Demo\")\n", - "print(f\"Dataset Created: {dataset.uid}\")\n", - "project.create_batches_from_dataset(\"batch\", dataset.uid)" + "If using NDJSON" ], - "cell_type": "code", - "outputs": [], - "execution_count": null + "cell_type": "markdown" }, { "metadata": {}, "source": [ - "## Prepare for upload\n", - "* Our local annotations need the following:\n", - " 1. signed url for segmentation masks\n", - " 2. data rows in labelbox\n", - " 3. feature schema ids" + "## Custom metrics" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "signer = lambda _bytes: client.upload_data(content=_bytes, sign=True)\n", - "predictions.add_url_to_masks(signer) \\\n", - " .add_url_to_data(signer) \\\n", - " .add_to_dataset(dataset, client.upload_data)" + "# Special case for custom feature metrics\n", + "\n", + "# NDJSON\n", + "radio_prediction_ndjson = {\n", + " 'name': 'radio_question',\n", + " 'answer': {'name': 'first_radio_answer', 'confidence': 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.1 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]}\n", + "}\n", + "\n", + "########## Nested Classifications are only supported with NDJSON tools ##########\n", + "\n", + "nested_radio_prediction_ndjson = {\n", + " \"name\": \"nested_radio_question\",\n", + " \"confidence\": 0.5 ,\n", + " \"answer\": { \"name\": \"first_radio_answer\", \"confidence\": 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ] },\n", + " \"classifications\" : [\n", + " {\n", + " \"name\": \"sub_radio_question\",\n", + " \"answer\": {\"name\": \"first_sub_radio_answer\", \"confidence\": 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ] }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "\n", + "nested_checklist_prediction_ndjson = {\n", + " \"name\": \"nested_checklist_question\",\n", + " \"confidence\": 0.5 ,\n", + " \"answer\": [{\n", + " \"name\": \"first_checklist_answer\",\n", + " \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"classifications\" : [\n", + " {\n", + " \"name\": \"sub_checklist_question\",\n", + " \"answer\": {\"name\": \"first_sub_checklist_answer\", \"confidence\": 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ] }\n", + " }\n", + " ]\n", + " }]\n", + "}\n", + "\n", + "############ Checklist ############\n", + "checklist_prediction_ndjson = {\n", + " 'name': 'checklist_question',\n", + " 'answer': [\n", + " {'name': 'first_checklist_answer' , 'confidence': 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]},\n", + " {'name': 'second_checklist_answer', 'confidence': 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]}\n", + " ]\n", + "}\n", + "all_annotations = [\n", + " radio_prediction_ndjson,\n", + " nested_radio_prediction_ndjson,\n", + " checklist_prediction_ndjson,\n", + " nested_checklist_prediction_ndjson,\n", + " ]\n", + "\n", + "three_k_annotations = all_annotations\n", + "\n", + "ndjson_prediction_method2 = []\n", + "dr_ids = [data_row.uid for data_row in dataset.export_data_rows()]\n", + "for data_row_id in dr_ids:\n", + " for annot in three_k_annotations:\n", + " ndjson_prediction_method2.append({\n", + " **annot,\n", + " 'uuid': str(uuid.uuid4()),\n", + " 'dataRow': {'id': data_row_id },\n", + " })" ], "cell_type": "code", "outputs": [], @@ -366,24 +787,64 @@ { "metadata": {}, "source": [ - "## **Optional** - Create labels with [Model Assisted Labeling](https://docs.labelbox.com/en/core-concepts/model-assisted-labeling)\n", - "\n", - "* Pre-label image so that we can quickly create ground truth\n", - "* Create ground truth data for Model Diagnostics\n", - "* Click on link below to label" + "## OCR special case" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "RUN_MAL = True\n", - "if RUN_MAL:\n", - " project.enable_model_assisted_labeling()\n", - " upload_task = lb.MALPredictionImport.create_from_objects(\n", - " client, project.uid, f'mal-import-{uuid.uuid4()}', predictions)\n", - " upload_task.wait_until_done()\n", - " print(upload_task.state, '\\n')" + "# Special case for OCR\n", + "\n", + "bbox_with_free_text_subclass_prediction_ndjson1 = {\n", + " \"name\": \"bbox_with_free_text_subclassification\",\n", + " \"confidence\": 0.5,\n", + " \"classifications\": [{\n", + " \"name\": \"sub_text_classification\",\n", + " \"confidence\": 0.5,\n", + " 'answer': 'aaaaaaaa - matching nicely',\n", + "\n", + " }],\n", + " \"bbox\": {\n", + " \"top\": 933,\n", + " \"left\": 541,\n", + " \"height\": 191,\n", + " \"width\": 330\n", + " }\n", + "}\n", + "bbox_with_free_text_subclass_prediction_ndjson2 = {\n", + " \"name\": \"bbox_with_free_text_subclassification\",\n", + " \"confidence\": 0.5,\n", + " \"classifications\": [{\n", + " \"name\": \"sub_text_classification\",\n", + " \"confidence\": 0.5,\n", + " 'answer': 'zzzzzzz',\n", + "\n", + " }],\n", + " \"bbox\": {\n", + " \"top\": 913,\n", + " \"left\": 521,\n", + " \"height\": 50,\n", + " \"width\": 50\n", + " }\n", + "}\n", + "all_annotations = [\n", + " bbox_with_free_text_subclass_prediction_ndjson1,\n", + " bbox_with_free_text_subclass_prediction_ndjson2,\n", + " ]\n", + "\n", + "three_k_annotations = []\n", + "three_k_annotations.extend(all_annotations)\n", + "\n", + "ndjson_prediction_method2 = []\n", + "dr_ids = [data_row.uid for data_row in dataset.export_data_rows()]\n", + "for data_row_id in dr_ids:\n", + " for annot in three_k_annotations:\n", + " ndjson_prediction_method2.append({\n", + " **annot,\n", + " 'uuid': str(uuid.uuid4()),\n", + " 'dataRow': {'id': data_row_id },\n", + " })" ], "cell_type": "code", "outputs": [], @@ -392,7 +853,34 @@ { "metadata": {}, "source": [ - "print(f\"https://app.labelbox.com/go-label/{project.uid}\")" + "\n", + "all_annotations = [\n", + " radio_prediction_ndjson,\n", + " checklist_prediction_ndjson,\n", + " bbox_prediction_ndjson,\n", + " bbox_with_radio_subclass_prediction_ndjson,\n", + " polygon_prediction_ndjson,\n", + " # mask_prediction_ndjson,\n", + " point_prediction_ndjson,\n", + " polyline_prediction_ndjson,\n", + " text_annotation_ndjson,\n", + " nested_radio_prediction_ndjson,\n", + " nested_checklist_prediction_ndjson\n", + " ]\n", + "\n", + "# three_k_annotations = bbox_predictions\n", + "# three_k_annotations.extend(all_annotations)\n", + "three_k_annotations = all_annotations\n", + "\n", + "ndjson_prediction_method2 = []\n", + "dr_ids = [data_row.uid for data_row in dataset.export_data_rows()]\n", + "for data_row_id in dr_ids:\n", + " for annot in three_k_annotations:\n", + " ndjson_prediction_method2.append({\n", + " **annot,\n", + " 'uuid': str(uuid.uuid4()),\n", + " 'dataRow': {'id': data_row_id }\n", + " })" ], "cell_type": "code", "outputs": [], @@ -401,19 +889,20 @@ { "metadata": {}, "source": [ - "## Export Labels\n", - "\n", - "We do not support `Skipped` labels and have a limit of **2000**" + "## Step 6. Upload the predictions payload to the Model Run" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "MAX_LABELS = 2000\n", - "labels = [\n", - " l for idx, l in enumerate(project.label_generator()) if idx < MAX_LABELS\n", - "]" + "# Upload the prediction label to the Model Run\n", + "upload_job_prediction = model_run.add_predictions(\n", + " name=\"prediction_upload_job\"+str(uuid.uuid4()),\n", + " predictions=ndjson_prediction_method2)\n", + "\n", + "# Errors will appear for prediction uploads that failed.\n", + "print(\"Errors:\", upload_job_prediction.errors)" ], "cell_type": "code", "outputs": [], @@ -422,16 +911,27 @@ { "metadata": {}, "source": [ - "## Setup Model & Model Run" + "## Step 7: Send annotations to a model run\n", + "To visualize both annotations and predictions in the model run we will create a project with ground truth annotations.\n", + "To send annotations to a Model Run, we must first import them into a project, create a label payload and then send them to the Model Run." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "##### 7.1. Create a labelbox project" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "lb_model = client.create_model(name=MODEL_NAME,\n", - " ontology_id=project.ontology().uid)\n", - "lb_model_run = lb_model.create_model_run(MODEL_VERSION)" + "# Create a Labelbox project\n", + "project = client.create_project(name=\"image_prediction_many_kinds\",\n", + " quality_mode=QualityMode.Benchmark,\n", + " media_type=MediaType.Image)\n", + "project.setup_editor(ontology)" ], "cell_type": "code", "outputs": [], @@ -440,14 +940,18 @@ { "metadata": {}, "source": [ - "Select label ids to upload" + "##### 7.2. Create a batch to send to the project" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "lb_model_run.upsert_labels([label.uid for label in labels])" + "project.create_batch(\n", + " \"batch_predictions_demo1\", # Each batch in a project must have a unique name\n", + " [data_row.uid for data_row in dataset.export_data_rows()], # A list of data rows or data row ids\n", + " 5 # priority between 1(Highest) - 5(lowest)\n", + ")" ], "cell_type": "code", "outputs": [], @@ -456,58 +960,188 @@ { "metadata": {}, "source": [ - "### Compute Metrics\n", - "* First get pairs of labels and predictions" + "##### 7.3 Create the annotations payload" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "* Create helper functions for our metric\n", - "* All functions will accept ground truth and prediction annotations" + "########### Annotations ###########\n", + "radio_annotation_ndjson = {\n", + " \"name\": \"radio_question\",\n", + " \"answer\": {\"name\": \"first_radio_answer\"}\n", + "}\n", + "\n", + "nested_radio_annotation_ndjson = {\n", + " \"name\": \"nested_radio_question\",\n", + " \"answer\": {\"name\": \"first_radio_answer\"},\n", + " \"classifications\" : [\n", + " {\"name\": \"sub_radio_question\", \"answer\": {\"name\": \"first_sub_radio_answer\"}}\n", + " ]\n", + "}\n", + "\n", + "checklist_annotation_ndjson = {\n", + " \"name\": \"checklist_question\",\n", + " \"answer\": [\n", + " {\"name\": \"first_checklist_answer\"},\n", + " {\"name\": \"second_checklist_answer\"}\n", + " ]\n", + "}\n", + "\n", + "bbox_annotation_ndjson = {\n", + " \"name\": \"bounding_box\",\n", + " \"bbox\": {\n", + " \"top\": 877,\n", + " \"left\": 1490,\n", + " \"height\": 130,\n", + " \"width\": 125\n", + " }\n", + "}\n", + "\n", + "bbox_with_radio_subclass_ndjson = {\n", + " \"name\": \"bbox_with_radio_subclass\",\n", + " \"classifications\": [{\n", + " \"name\": \"sub_radio_question\",\n", + " \"answer\":\n", + " { \"name\":\"first_sub_radio_answer\" }\n", + "\n", + " }],\n", + " \"bbox\": {\n", + " \"top\": 533,\n", + " \"left\": 841,\n", + " \"height\": 191,\n", + " \"width\": 230\n", + " }\n", + "}\n", + "\n", + "polygon_annotation_ndjson = {\n", + " \"name\": \"polygon\",\n", + " \"polygon\": [\n", + " {\"x\": 1489.581, \"y\": 183.934},\n", + " {\"x\": 2278.306, \"y\": 256.885},\n", + " {\"x\": 2428.197, \"y\": 200.437},\n", + " {\"x\": 2560.0, \"y\": 335.419},\n", + " {\"x\": 2557.386, \"y\": 503.165},\n", + " {\"x\": 2320.596, \"y\": 503.103},\n", + " {\"x\": 2156.083, \"y\": 628.943},\n", + " {\"x\": 2161.111, \"y\": 785.519},\n", + " {\"x\": 2002.115, \"y\": 894.647},\n", + " {\"x\": 1838.456, \"y\": 877.874},\n", + " {\"x\": 1436.53, \"y\": 874.636},\n", + " {\"x\": 1411.403, \"y\": 758.579},\n", + " {\"x\": 1353.853, \"y\": 751.74},\n", + " {\"x\": 1345.264, \"y\": 453.461},\n", + " {\"x\": 1426.011, \"y\": 421.129},\n", + " {\"x\": 1489.581, \"y\": 183.934}\n", + " ]\n", + "}\n", + "\n", + "mask_annotation_ndjson = {\n", + " \"name\": \"mask\",\n", + " \"classifications\": [],\n", + " \"mask\": {\"instanceURI\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png\",\n", + " \"colorRGB\": (0, 0, 0)}\n", + "}\n", + "\n", + "\n", + "point_annotation_ndjson = {\n", + " \"name\": \"point\",\n", + " \"classifications\": [],\n", + " \"point\": {\"x\": 1166.606, \"y\": 1441.768}\n", + "}\n", + "\n", + "point_annotation_ndjson = {\n", + " \"name\": \"point\",\n", + " \"classifications\": [],\n", + " \"point\": {\"x\": 1166.606, \"y\": 1441.768}\n", + "}\n", + "\n", + "polyline_annotation_ndjson = {\n", + " \"name\": \"polyline\",\n", + " \"classifications\": [],\n", + " \"line\": [\n", + " {\"x\": 2534.353, \"y\": 249.471},\n", + " {\"x\": 2429.492, \"y\": 182.092},\n", + " {\"x\": 2294.322, \"y\": 221.962},\n", + " {\"x\": 2224.491, \"y\": 180.463},\n", + " {\"x\": 2136.123, \"y\": 204.716},\n", + " {\"x\": 1712.247, \"y\": 173.949},\n", + " {\"x\": 1703.838, \"y\": 84.438},\n", + " {\"x\": 1579.772, \"y\": 82.61},\n", + " {\"x\": 1583.442, \"y\": 167.552},\n", + " {\"x\": 1478.869, \"y\": 164.903},\n", + " {\"x\": 1418.941, \"y\": 318.149},\n", + " {\"x\": 1243.128, \"y\": 400.815},\n", + " {\"x\": 1022.067, \"y\": 319.007},\n", + " {\"x\": 892.367, \"y\": 379.216},\n", + " {\"x\": 670.273, \"y\": 364.408},\n", + " {\"x\": 613.114, \"y\": 288.16},\n", + " {\"x\": 377.559, \"y\": 238.251},\n", + " {\"x\": 368.087, \"y\": 185.064},\n", + " {\"x\": 246.557, \"y\": 167.286},\n", + " {\"x\": 236.648, \"y\": 285.61},\n", + " {\"x\": 90.929, \"y\": 326.412}\n", + " ]\n", + "}\n", + "\n", + "nested_checklist_annotation_ndjson = {\n", + " \"name\": \"nested_checklist_question\",\n", + " \"answer\": [{\n", + " \"name\": \"first_checklist_answer\",\n", + " \"classifications\" : [\n", + " {\n", + " \"name\": \"sub_checklist_question\",\n", + " \"answer\": {\"name\": \"first_sub_checklist_answer\"}\n", + " }\n", + " ]\n", + " }]\n", + "}\n", + "\n", + "text_annotation_ndjson = {\n", + " \"name\": \"free_text\",\n", + " \"answer\": \"sample text\",\n", + "}\n" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "##### 7.4. Create the label object" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "from shapely.ops import cascaded_union\n", - "\n", - "\n", - "def nearby_cars_iou(ground_truths, predictions, area_threshold=17000):\n", - " \"\"\"\n", - " Metric to track the iou score for cars that are nearby (determined by pixel size).\n", - " \n", - " This might be useful to investigate why the model poorly when vehicles are nearby.\n", - " We might care a lot about optimizing this metric because our self driving car needs to \n", - " be aware of its immediate surroundings for safety reasons.\n", - " \"\"\"\n", - " ground_truths = [gt for gt in ground_truths if gt.name == 'car']\n", - " predictions = [pred for pred in predictions if pred.name == 'car']\n", - " ground_truths = cascaded_union([\n", - " gt.value.shapely\n", - " for gt in ground_truths\n", - " if gt.value.shapely.area > area_threshold\n", - " ])\n", - " predictions = cascaded_union([\n", - " pred.value.shapely\n", - " for pred in predictions\n", - " if pred.value.shapely.area > area_threshold\n", - " ])\n", - " union = ground_truths.union(predictions).area\n", - " # If there is no prediction or label then the score is undefined\n", - " if union == 0:\n", - " return []\n", - " return [\n", - " ScalarMetric(\n", - " value=ground_truths.intersection(predictions).area / union,\n", - " metric_name=\"iou\",\n", - " feature_name=\"car\",\n", - " subclass_name=\n", - " \"nearby\" # Doesn't necessarily need to be a subclass in the ontology\n", - " )\n", - " ]" + "# Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations\n", + "annotations = [\n", + " radio_annotation_ndjson,\n", + " checklist_annotation_ndjson,\n", + " bbox_annotation_ndjson,\n", + " bbox_with_radio_subclass_ndjson,\n", + " polygon_annotation_ndjson,\n", + " mask_annotation_ndjson,\n", + " point_annotation_ndjson,\n", + " polyline_annotation_ndjson,\n", + " nested_radio_annotation_ndjson,\n", + " nested_checklist_annotation_ndjson,\n", + " text_annotation_ndjson\n", + " ]\n", + "# gts = bbox_annotations\n", + "gts = (annotations)\n", + "ndjson_annotation = []\n", + "for data_row in dataset.export_data_rows():\n", + " for annot in gts:\n", + " ndjson_annotation.append({\n", + " **annot,\n", + " 'uuid': str(uuid.uuid4()),\n", + " 'dataRow': {'id': data_row.uid}\n", + " })\n", + "\n" ], "cell_type": "code", "outputs": [], @@ -516,26 +1150,22 @@ { "metadata": {}, "source": [ - "* Compute and sssign each metric to prediction label" + "##### 7.5. Upload annotations to the project using Label Import" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "# Metric functions expect to be provided labels and predictions that correspond to the same data image/video frame\n", - "# This get_label_pairs function uses the datarow id to achieve this.\n", - "pairs = get_label_pairs(labels, predictions, filter_mismatch=True)\n", - "for (ground_truth, prediction) in pairs.values():\n", - " metrics = []\n", - " metrics.extend(\n", - " feature_miou_metric(ground_truth.annotations, prediction.annotations))\n", - " metrics.extend(\n", - " feature_confusion_matrix_metric(ground_truth.annotations,\n", - " prediction.annotations))\n", - " metrics.extend(\n", - " nearby_cars_iou(ground_truth.annotations, prediction.annotations))\n", - " prediction.annotations.extend(metrics)" + "upload_job_annotation = LabelImport.create_from_objects(\n", + " client = client,\n", + " project_id = project.uid,\n", + " name=\"annotation_import_\" + str(uuid.uuid4()),\n", + " labels=ndjson_annotation)\n", + "\n", + "upload_job_annotation.wait_until_done()\n", + "# Errors will appear for annotation uploads that failed.\n", + "print(\"Errors:\", upload_job_annotation.errors)\n" ], "cell_type": "code", "outputs": [], @@ -544,17 +1174,18 @@ { "metadata": {}, "source": [ - "### Upload to Labelbox" + "##### 7.6 Send the annotations to the Model Run" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "upload_task = lb_model_run.add_predictions(\n", - " f'diagnostics-import-{uuid.uuid4()}', predictions)\n", - "upload_task.wait_until_done()\n", - "print(upload_task.state)" + "# get the labels id from the project\n", + "task = project.export_v2()\n", + "task.wait_till_done()\n", + "label_ids = [l[\"id\"] for dr in task.result for l in dr[\"projects\"][project.uid][\"labels\"]]\n", + "model_run.upsert_labels(label_ids)" ], "cell_type": "code", "outputs": [], @@ -563,17 +1194,15 @@ { "metadata": {}, "source": [ - "### Open Model Run" + "## Optional deletions for cleanup\n" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "for idx, model_run_data_row in enumerate(lb_model_run.model_run_data_rows()):\n", - " if idx == 5:\n", - " break\n", - " print(model_run_data_row.url)" + "# project.delete()\n", + "# dataset.delete()" ], "cell_type": "code", "outputs": [], From 0452243be49a899a6c4f95c12ed97c2be5f8e83e Mon Sep 17 00:00:00 2001 From: ovalle15 Date: Tue, 27 Feb 2024 19:04:15 -0500 Subject: [PATCH 2/4] AO updates to custom_metrics --- .../custom_metrics_demo.ipynb | 1410 ++++++++--------- examples/project_configuration/webhooks.ipynb | 199 +-- 2 files changed, 722 insertions(+), 887 deletions(-) diff --git a/examples/model_diagnostics/custom_metrics_demo.ipynb b/examples/model_diagnostics/custom_metrics_demo.ipynb index 593395298..b52eaaa23 100644 --- a/examples/model_diagnostics/custom_metrics_demo.ipynb +++ b/examples/model_diagnostics/custom_metrics_demo.ipynb @@ -17,12 +17,12 @@ "source": [ "\n", "\n", - "\n", "\n", "\n", "\n", - "\n", "" ], @@ -61,22 +61,10 @@ { "metadata": {}, "source": [ - "from labelbox.schema.ontology import OntologyBuilder, Tool, Classification, Option\n", - "from labelbox import Client, MALPredictionImport, LabelImport\n", - "from labelbox.data.serialization import NDJsonConverter\n", - "from labelbox.schema.media_type import MediaType\n", - "from labelbox.schema.quality_mode import QualityMode\n", - "from labelbox.data.metrics.group import get_label_pairs\n", - "from labelbox.data.metrics import feature_miou_metric, feature_confusion_matrix_metric\n", - "from labelbox.data.annotation_types import (\n", - " Label, ImageData, ObjectAnnotation, MaskData,\n", - " Rectangle, Point, Line, Mask, Polygon,\n", - " Radio, Checklist, Text,\n", - " ClassificationAnnotation, ClassificationAnswer, RelationshipAnnotation, Relationship\n", - ")\n", "import uuid\n", - "import numpy as np\n", - "from labelbox.schema.queue_mode import QueueMode" + "import requests\n", + "import labelbox as lb\n", + "import labelbox.types as lb_types" ], "cell_type": "code", "outputs": [], @@ -95,7 +83,7 @@ "source": [ "API_KEY = \"\"\n", "\n", - "client = Client(API_KEY)" + "client = lb.Client(API_KEY)" ], "cell_type": "code", "outputs": [], @@ -111,330 +99,435 @@ { "metadata": {}, "source": [ - "########### Radio Classification ###########\n", - "\n", - "# Python annotation\n", - "radio_prediction = ClassificationAnnotation(\n", - " name=\"radio_question\",\n", - " value=Radio(answer = ClassificationAnswer(name = \"first_radio_answer\", confidence=0.5))\n", - ")\n", + "### Classification: Radio (single-choice)" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "\n", "# NDJSON\n", "radio_prediction_ndjson = {\n", - " 'name': 'radio_question',\n", - " 'answer': {'name': 'first_radio_answer', 'confidence': 0.5}\n", - "}\n", - "\n", - "########## Nested Classifications are only supported with NDJSON tools ##########\n", - "\n", + " \"name\": \"radio_question\",\n", + " \"answer\": {'name': 'first_radio_answer', 'confidence': 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.1 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]\n", + " }\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Classification: checklist (multi-choice)" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "checklist_prediction_ndjson = {\n", + " 'name': 'checklist_question',\n", + " 'answer': [\n", + " {'name': 'first_checklist_answer' , 'confidence': 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]},\n", + " {'name': 'second_checklist_answer', 'confidence': 0.5, 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]}\n", + " ]\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Classification: Nested radio and checklist" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "nested_radio_prediction_ndjson = {\n", " \"name\": \"nested_radio_question\",\n", " \"confidence\": 0.5 ,\n", - " \"answer\": { \"name\": \"first_radio_answer\", \"confidence\": 0.5 },\n", + " \"answer\": { \"name\": \"first_radio_answer\", \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]\n", + " },\n", " \"classifications\" : [\n", " {\n", " \"name\": \"sub_radio_question\",\n", - " \"answer\": {\"name\": \"first_sub_radio_answer\", \"confidence\": 0.5 }\n", + " \"answer\": { \"name\": \"first_sub_radio_answer\", \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]\n", + " }\n", " }\n", - " ]\n", + " ]\n", "}\n", "\n", - "\n", "nested_checklist_prediction_ndjson = {\n", " \"name\": \"nested_checklist_question\",\n", " \"confidence\": 0.5 ,\n", " \"answer\": [{\n", " \"name\": \"first_checklist_answer\",\n", " \"confidence\": 0.5,\n", - " \"classifications\" : [\n", - " {\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"classifications\" : [{\n", " \"name\": \"sub_checklist_question\",\n", - " \"answer\": {\"name\": \"first_sub_checklist_answer\", \"confidence\": 0.5 }\n", + " \"answer\": {\n", + " \"name\": \"first_sub_checklist_answer\", \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ]}\n", " }\n", " ]\n", " }]\n", - "}\n", - "\n", - "############ Checklist ############\n", - "\n", - "# Python Annotations\n", - "checklist_prediction = ClassificationAnnotation(\n", - " name=\"checklist_question\", # must match your ontology feature's name\n", - " value=Checklist(\n", - " answer = [\n", - " ClassificationAnswer(\n", - " name = \"first_checklist_answer\",\n", - " confidence=0.5\n", - " ),\n", - " ClassificationAnswer(\n", - " name = \"second_checklist_answer\",\n", - " confidence=0.5\n", - " )\n", - " ]\n", - " )\n", - " )\n", - "\n", - "# NDJSON\n", - "checklist_prediction_ndjson = {\n", - " 'name': 'checklist_question',\n", - " 'answer': [\n", - " {'name': 'first_checklist_answer' , 'confidence': 0.5},\n", - " {'name': 'second_checklist_answer', 'confidence': 0.5}\n", - " ]\n", - "}\n", - "\n", - "####### Bounding box #######\n", - "\n", - "\n", - "# Python Annotation\n", - "bbox_prediction = ObjectAnnotation(\n", - " name = \"bounding_box\", # must match your ontology feature's name\n", - " confidence=0.5,\n", - " value=Rectangle(\n", - " start=Point(x=977, y=1690), # Top left\n", - " end=Point(x=330, y=225), # Bottom right\n", - " ),\n", - "\n", - ")\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Bounding Box" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "\n", - "#NDJSON\n", "bbox_prediction_ndjson = {\n", - " 'name': 'bounding_box',\n", - " 'confidence': 0.5,\n", - " 'bbox': {\n", - " \"top\": 500,\n", - " \"left\": 1190,\n", - " \"height\": 230,\n", + " \"name\": \"bounding_box\",\n", + " \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"bbox\": {\n", + " \"top\": 977,\n", + " \"left\": 1690,\n", + " \"height\": 330,\n", " \"width\": 225\n", " }\n", - "}\n", - "\n", - "####### Bounding box with nested classification #######\n", - "bbox_with_radio_subclass_prediction = ObjectAnnotation(\n", - " name=\"bbox_with_radio_subclass\",\n", - " confidence=0.5, # must match your ontology feature's name\n", - " value=Rectangle(\n", - " start=Point(x=433, y=341), # Top left\n", - " end=Point(x=291, y=230), # Bottom right\n", - " ),\n", - " classifications=[\n", - " \tClassificationAnnotation(\n", - " \tname=\"sub_radio_question\",\n", - " \t\tvalue=Radio(answer=ClassificationAnswer(name=\"first_sub_radio_answer\", confidence=0.5))\n", - " )\n", - " ]\n", - ")\n", - "\n", - "\n", + "}\n" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Bounding box with nested classification " + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "## NDJSON\n", "bbox_with_radio_subclass_prediction_ndjson = {\n", " \"name\": \"bbox_with_radio_subclass\",\n", " \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.2 },\n", + " { 'name': 'precision', 'value': 0.1 },\n", + " { 'name': 'recall', 'value': 0.3 },\n", + " { 'name': 'tagsCount', 'value': 23 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", " \"classifications\": [{\n", " \"name\": \"sub_radio_question\",\n", - " \"confidence\": 0.5,\n", " \"answer\":\n", " { \"name\":\"first_sub_radio_answer\", \"confidence\": 0.5}\n", "\n", " }],\n", " \"bbox\": {\n", - " \"top\": 533,\n", - " \"left\": 641,\n", - " \"height\": 291,\n", - " \"width\": 330\n", - " }\n", - "}\n", - "\n", - "\n", - "####### Bounding box with nested free text #######\n", - "\n", - "## NDJSON\n", - "bbox_with_free_text_subclass_prediction_ndjson = {\n", - " \"name\": \"bbox_with_free_text_subclass\",\n", - " \"confidence\": 0.5,\n", - " \"classifications\": [{\n", - " \"name\": \"text\",\n", - " \"confidence\": 0.5,\n", - " 'answer': 'sample text',\n", - "\n", - " }],\n", - " \"bbox\": {\n", " \"top\": 933,\n", " \"left\": 541,\n", " \"height\": 191,\n", " \"width\": 330\n", " }\n", - "}\n", - "\n", - "########## Polygon ##########\n", - "# Python Anotation\n", - "polygon_prediction = ObjectAnnotation(\n", - " name = \"polygon\", # must match your ontology feature's name\n", - " confidence = 0.5,\n", - " value=Polygon( # Coordinates for the verticies of your polygon\n", - " points=[Point(x=1489.581,y=183.934),Point(x=2278.306,y=256.885),Point(x=2428.197,y=200.437),Point(x=2560.0,y=335.419),\n", - " Point(x=2557.386,y=503.165),Point(x=2320.596,y=503.103),Point(x=2156.083, y=628.943),Point(x=2161.111,y=785.519),\n", - " Point(x=2002.115, y=894.647),Point(x=1838.456,y=877.874),Point(x=1436.53,y=874.636),Point(x=1411.403,y=758.579),\n", - " Point(x=1353.853,y=751.74),Point(x=1345.264, y=453.461),Point(x=1426.011,y=421.129)]\n", - " ),\n", - ")\n", - "\n", - "polygon_prediction_target = ObjectAnnotation(\n", - " name = \"polygon\", # must match your ontology feature's name\n", - " confidence = 0.5,\n", - " value=Polygon( # Coordinates for the verticies of your polygon\n", - " points=[Point(x=1089.581,y=203.934),Point(x=1878.306,y=276.885),Point(x=2028.197,y=220.437),Point(x=2160.0,y=355.419),\n", - " Point(x=2157.386,y=523.165),Point(x=2020.596,y=523.103),Point(x=1756.083, y=648.943),Point(x=1761.111,y=805.519),\n", - " Point(x=1602.115, y=914.647),Point(x=1438.456,y=897.874),Point(x=1036.53,y=894.636),Point(x=1011.403,y=778.579),\n", - " Point(x=953.853,y=771.74),Point(x=945.264, y=473.461),Point(x=926.011,y=441.129)]\n", - " ),\n", - ")\n", - "\n", - "polygon_relationship = RelationshipAnnotation(\n", - " name=\"relationship\",\n", - " value=Relationship(\n", - " source=polygon_prediction,\n", - " target=polygon_prediction_target,\n", - " type=Relationship.Type.UNIDIRECTIONAL,\n", - " ))\n", - "\n", - "\n", - "# NDJSON\n", - "\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Polygon" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "polygon_prediction_ndjson = {\n", - " 'name': 'polygon',\n", - " 'confidence': 0.5,\n", - " 'polygon': [\n", - " {'x': 1489.581, 'y': 183.934},\n", - " {'x': 2278.306, 'y': 256.885},\n", - " {'x': 2428.197, 'y': 200.437},\n", - " {'x': 2560.0, 'y': 335.419},\n", - " {'x': 2557.386, 'y': 503.165},\n", - " {'x': 2320.596, 'y': 503.103},\n", - " {'x': 2156.083, 'y': 628.943},\n", - " {'x': 2161.111, 'y': 785.519},\n", - " {'x': 2002.115, 'y': 894.647},\n", - " {'x': 1838.456, 'y': 877.874},\n", - " {'x': 1436.53, 'y': 874.636},\n", - " {'x': 1411.403, 'y': 758.579},\n", - " {'x': 1353.853, 'y': 751.74},\n", - " {'x': 1345.264, 'y': 453.461},\n", - " {'x': 1426.011, 'y': 421.129},\n", - " {'x': 1489.581, 'y': 183.934}\n", + " \"name\": \"polygon\",\n", + " \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"polygon\": [\n", + " {\"x\": 1489.581, \"y\": 183.934},\n", + " {\"x\": 2278.306, \"y\": 256.885},\n", + " {\"x\": 2428.197, \"y\": 200.437},\n", + " {\"x\": 2560.0, \"y\": 335.419},\n", + " {\"x\": 2557.386, \"y\": 503.165},\n", + " {\"x\": 2320.596, \"y\": 503.103},\n", + " {\"x\": 2156.083, \"y\": 628.943},\n", + " {\"x\": 2161.111, \"y\": 785.519},\n", + " {\"x\": 2002.115, \"y\": 894.647},\n", + " {\"x\": 1838.456, \"y\": 877.874},\n", + " {\"x\": 1436.53, \"y\": 874.636},\n", + " {\"x\": 1411.403, \"y\": 758.579},\n", + " {\"x\": 1353.853, \"y\": 751.74},\n", + " {\"x\": 1345.264, \"y\": 453.461},\n", + " {\"x\": 1426.011, \"y\": 421.129},\n", + " {\"x\": 1489.581, \"y\": 183.934}\n", " ]\n", - "}\n", - "\n", - "####### Free text #######\n", - "# Confidence is not supported for text prediction\n", - "# Python annotation\n", - "text_annotation = ClassificationAnnotation(\n", - " name=\"free_text\", # must match your ontology feature's name\n", - " value=Text(answer=\"sample text\")\n", - ")\n", - "\n", - "# NDJSON\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Classification: Free-form text" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "text_annotation_ndjson = {\n", - " 'name': 'free_text',\n", - " 'answer': 'sample text',\n", - "}\n", - "\n", - "######### Segmentation mask #########\n", - "\n", - "# Python\n", - "# Identifying what values in the numpy array correspond to the mask annotation\n", - "color = (0, 0, 0)\n", - "\n", - "# convert a polygon to mask\n", - "im_height, im_width = 100,100 #need to provide the height and width of image.\n", - "mask_data = MaskData(arr=\n", - " polygon_prediction.value.draw(height=im_height,width=im_width,color=color))\n", - "\n", - "# convert a 2D array to 3D array\n", - "arr_2d = np.zeros((100,100), dtype='uint8')\n", - "mask_data = MaskData.from_2D_arr(arr_2d)\n", - "\n", - "# a 3D array where 3rd axis is RGB values.\n", - "mask_data = MaskData(arr= np.zeros([400,450,3],dtype='uint8'))\n", - "\n", - "mask_prediction = ObjectAnnotation(\n", - " name = \"mask\", # must match your ontology feature's name\n", - " confidence=0.5,\n", - " value=Mask(mask=mask_data, color=color),\n", - ")\n", + " \"name\": \"free_text\",\n", + " \"answer\": \"sample text\",\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"confidence\": 0.5\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Segmentation mask" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "### Raster Segmentation (Byte string array)\n", + "url = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png\"\n", "\n", "mask_prediction_ndjson = {\n", " \"name\": \"mask\",\n", " \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", " \"classifications\": [],\n", - " \"mask\": {\"instanceURI\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png\",\n", + " \"mask\": {\"instanceURI\": url,\n", " \"colorRGB\": (255, 255, 255)}\n", "}\n", - "\n", - "######## Point ########\n", - "\n", - "# Python Annotation\n", - "point_prediction = ObjectAnnotation(\n", - " name = \"point\", # must match your ontology feature's name\n", - " confidence=0.5,\n", - " value = Point(x=1166.606, y=1441.768),\n", - ")\n", - "\n", - "\n", - "# NDJSON\n", + "\n" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Segmentation mask with nested classification" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "url_2 = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg_with_subclass.png\"\n", + "\n", + "# NDJSON using instanceURI, bytes array is not fully supported.\n", + "mask_with_text_subclass_prediction_ndjson = {\n", + " \"name\": \"mask_with_text_subclass\",\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"mask\": {\"instanceURI\": url_2,\n", + " \"colorRGB\": (255, 255, 255)},\n", + " \"classifications\":[{\n", + " \"name\": \"sub_free_text\",\n", + " \"answer\": \"free text answer\"\n", + " }]\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Point" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "point_prediction_ndjson = {\n", - " 'name': 'point',\n", - " 'confidence': 0.5,\n", - " 'classifications': [],\n", - " 'point': {'x': 1166.606, 'y': 1441.768}\n", - "}\n", - "\n", - "###### Polyline ######\n", - "\n", - "\n", - "# Python Annotation\n", - "\n", - "polyline_prediction = ObjectAnnotation(\n", - " name = \"polyline\", # must match your ontology feature's name\n", - " confidence=0.5, ## Not supported for python annotation tools\n", - " value=Line( # Coordinates for the keypoints in your polyline\n", - " points=[Point(x=2534.353, y=249.471),Point(x=2429.492, y=182.092),Point(x=2294.322, y=221.962),Point(x=2224.491, y=180.463),Point(x=2136.123, y=204.716),\n", - " Point(x=1712.247, y=173.949),Point(x=1703.838, y=84.438),Point(x=1579.772, y=82.61),Point(x=1583.442, y=167.552),\n", - " Point(x=1478.869, y=164.903),Point(x=1418.941, y=318.149),Point(x=1243.128, y=400.815),Point(x=1022.067, y=319.007),\n", - " Point(x=892.367, y=379.216),Point(x=670.273, y=364.408),Point(x=613.114, y=288.16),Point(x=377.559, y=238.251),\n", - " Point(x=368.087, y=185.064),Point(x=246.557, y=167.286),Point(x=236.648, y=285.61),Point(x=90.929, y=326.412)]\n", - " ),\n", - ")\n", - "\n", - "# NDJSON\n", + " \"name\": \"point\",\n", + " \"confidence\": 0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"classifications\": [],\n", + " \"point\": {\"x\": 1166.606, \"y\": 1441.768}\n", + "}" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Polyline" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ "polyline_prediction_ndjson = {\n", - " 'name': 'polyline',\n", - " 'confidence':0.5,\n", - " 'classifications': [],\n", - " 'line': [\n", - " {'x': 2534.353, 'y': 249.471},\n", - " {'x': 2429.492, 'y': 182.092},\n", - " {'x': 2294.322, 'y': 221.962},\n", - " {'x': 2224.491, 'y': 180.463},\n", - " {'x': 2136.123, 'y': 204.716},\n", - " {'x': 1712.247, 'y': 173.949},\n", - " {'x': 1703.838, 'y': 84.438},\n", - " {'x': 1579.772, 'y': 82.61},\n", - " {'x': 1583.442, 'y': 167.552},\n", - " {'x': 1478.869, 'y': 164.903},\n", - " {'x': 1418.941, 'y': 318.149},\n", - " {'x': 1243.128, 'y': 400.815},\n", - " {'x': 1022.067, 'y': 319.007},\n", - " {'x': 892.367, 'y': 379.216},\n", - " {'x': 670.273, 'y': 364.408},\n", - " {'x': 613.114, 'y': 288.16},\n", - " {'x': 377.559, 'y': 238.251},\n", - " {'x': 368.087, 'y': 185.064},\n", - " {'x': 246.557, 'y': 167.286},\n", - " {'x': 236.648, 'y': 285.61},\n", - " {'x': 90.929, 'y': 326.412}\n", + " \"name\": \"polyline\",\n", + " \"confidence\":0.5,\n", + " 'customMetrics': [\n", + " { 'name': 'iou', 'value': 0.5 },\n", + " { 'name': 'f1', 'value': 0.33 },\n", + " { 'name': 'precision', 'value': 0.55 },\n", + " { 'name': 'recall', 'value': 0.33 },\n", + " { 'name': 'tagsCount', 'value': 43 },\n", + " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", + " ],\n", + " \"classifications\": [],\n", + " \"line\": [\n", + " {\"x\": 2534.353, \"y\": 249.471},\n", + " {\"x\": 2429.492, \"y\": 182.092},\n", + " {\"x\": 2294.322, \"y\": 221.962},\n", + " {\"x\": 2224.491, \"y\": 180.463},\n", + " {\"x\": 2136.123, \"y\": 204.716},\n", + " {\"x\": 1712.247, \"y\": 173.949},\n", + " {\"x\": 1703.838, \"y\": 84.438},\n", + " {\"x\": 1579.772, \"y\": 82.61},\n", + " {\"x\": 1583.442, \"y\": 167.552},\n", + " {\"x\": 1478.869, \"y\": 164.903},\n", + " {\"x\": 1418.941, \"y\": 318.149},\n", + " {\"x\": 1243.128, \"y\": 400.815},\n", + " {\"x\": 1022.067, \"y\": 319.007},\n", + " {\"x\": 892.367, \"y\": 379.216},\n", + " {\"x\": 670.273, \"y\": 364.408},\n", + " {\"x\": 613.114, \"y\": 288.16},\n", + " {\"x\": 377.559, \"y\": 238.251},\n", + " {\"x\": 368.087, \"y\": 185.064},\n", + " {\"x\": 246.557, \"y\": 167.286},\n", + " {\"x\": 236.648, \"y\": 285.61},\n", + " {\"x\": 90.929, \"y\": 326.412}\n", " ]\n", - "}\n", - "\n" + "}\n" ], "cell_type": "code", "outputs": [], @@ -450,16 +543,25 @@ { "metadata": {}, "source": [ - "NUM_DATA_ROWS = 1\n", - "\n", "# send a sample image as batch to the project\n", - "test_img_urls = [{\n", + "global_key = \"2560px-Kitano_Street_Kobe01s5s4110.jpeg\"\n", + "test_img_urls = {\n", " \"row_data\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg\",\n", - " \"global_key\": str(uuid.uuid4())\n", - "} for i in range(0, NUM_DATA_ROWS)]\n", - "dataset = client.create_dataset(name=\"image_prediction_demo\", iam_integration=None)\n", - "data_rows = dataset.create_data_rows(test_img_urls)\n", - "print(data_rows)" + " \"global_key\": global_key\n", + "}\n", + "\n", + "dataset = client.create_dataset(name=\"Custom metrics demo\", iam_integration=None)\n", + "task = dataset.create_data_rows([test_img_urls])\n", + "\n", + "print(f\"Failed data rows: {task.failed_data_rows}\")\n", + "print(f\"Errors: {task.errors}\")\n", + "\n", + "if task.errors:\n", + " for error in task.errors:\n", + " if 'Duplicate global key' in error['message'] and dataset.row_count == 0:\n", + " # If the global key already exists in the workspace the dataset will be created empty, so we can delete it.\n", + " print(f\"Deleting empty dataset: {dataset}\")\n", + " dataset.delete()" ], "cell_type": "code", "outputs": [], @@ -476,173 +578,103 @@ { "metadata": {}, "source": [ - "ontology_builder = OntologyBuilder(\n", + "ontology_builder = lb.OntologyBuilder(\n", " classifications=[ # List of Classification objects\n", - " Classification( # Radio classification given the name \"text\" with two options: \"first_radio_answer\" and \"second_radio_answer\"\n", - " class_type=Classification.Type.RADIO,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", " name=\"radio_question\",\n", " options=[\n", - " Option(value=\"first_radio_answer\"),\n", - " Option(value=\"second_radio_answer\")\n", + " lb.Option(value=\"first_radio_answer\"),\n", + " lb.Option(value=\"second_radio_answer\")\n", " ]\n", " ),\n", - " Classification( # Checklist classification given the name \"text\" with two options: \"first_checklist_answer\" and \"second_checklist_answer\"\n", - " class_type=Classification.Type.CHECKLIST,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.CHECKLIST,\n", " name=\"checklist_question\",\n", " options=[\n", - " Option(value=\"first_checklist_answer\"),\n", - " Option(value=\"second_checklist_answer\")\n", + " lb.Option(value=\"first_checklist_answer\"),\n", + " lb.Option(value=\"second_checklist_answer\")\n", " ]\n", " ),\n", - " Classification( # Text classification given the name \"text\"\n", - " class_type=Classification.Type.TEXT,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.TEXT,\n", " name=\"free_text\"\n", " ),\n", - " Classification(\n", - " class_type=Classification.Type.RADIO,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", " name=\"nested_radio_question\",\n", " options=[\n", - " Option(\"first_radio_answer\",\n", + " lb.Option(\"first_radio_answer\",\n", " options=[\n", - " Classification(\n", - " class_type=Classification.Type.RADIO,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", " name=\"sub_radio_question\",\n", - " options=[Option(\"first_sub_radio_answer\")]\n", + " options=[lb.Option(\"first_sub_radio_answer\")]\n", " )\n", " ]\n", " )\n", " ]\n", " ),\n", - " Classification(\n", - " class_type=Classification.Type.CHECKLIST,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.CHECKLIST,\n", " name=\"nested_checklist_question\",\n", " options=[\n", - " Option(\"first_checklist_answer\",\n", + " lb.Option(\"first_checklist_answer\",\n", " options=[\n", - " Classification(\n", - " class_type=Classification.Type.CHECKLIST,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.CHECKLIST,\n", " name=\"sub_checklist_question\",\n", - " options=[Option(\"first_sub_checklist_answer\")]\n", + " options=[lb.Option(\"first_sub_checklist_answer\")]\n", " )\n", " ]\n", " )\n", " ]\n", " ),\n", " ],\n", - " tools=[ # List of Tool objects\n", - " Tool( # Bounding Box tool given the name \"box\"\n", - " tool=Tool.Type.BBOX,\n", + " tools=[ # List of tools\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.BBOX,\n", " name=\"bounding_box\"),\n", - " Tool( # Bounding Box tool given the name \"box\"\n", - " tool=Tool.Type.BBOX,\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.BBOX,\n", " name=\"bbox_with_radio_subclass\",\n", " classifications=[\n", - " Classification(\n", - " class_type=Classification.Type.RADIO,\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", " name=\"sub_radio_question\",\n", " options=[\n", - " Option(value=\"first_sub_radio_answer\")\n", + " lb.Option(value=\"first_sub_radio_answer\")\n", " ]\n", " ),\n", " ]\n", " ),\n", - " Tool( # Bounding Box tool given the name \"box\"\n", - " tool=Tool.Type.BBOX,\n", - " name=\"bbox_with_free_text_subclassification\",\n", - " classifications=[\n", - " Classification(\n", - " class_type=Classification.Type.TEXT,\n", - " name=\"sub_text_classification\",\n", - " ),\n", - " ]\n", - " ),\n", - " Tool( # Polygon tool given the name \"polygon\"\n", - " tool=Tool.Type.POLYGON,\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.POLYGON,\n", " name=\"polygon\"),\n", - " Tool( # Segmentation mask tool given the name \"mask\"\n", - " tool=Tool.Type.RASTER_SEGMENTATION,\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.RASTER_SEGMENTATION,\n", " name=\"mask\"),\n", - " \t Tool( # Point tool given the name \"point\"\n", - " tool=Tool.Type.POINT,\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.RASTER_SEGMENTATION,\n", + " name=\"mask_with_text_subclass\",\n", + " classifications=[\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.TEXT,\n", + " name=\"sub_free_text\")\n", + " ]\n", + " ),\n", + " \t lb.Tool(\n", + " tool=lb.Tool.Type.POINT,\n", " name=\"point\"),\n", - " Tool( # Polyline tool given the name \"line\"\n", - " tool=Tool.Type.LINE,\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.LINE,\n", " name=\"polyline\")]\n", - ")\n", - "\n", - "ontology = client.create_ontology(\"Image Prediction Import Demo\", ontology_builder.asdict(), media_type=MediaType.Image)" - ], - "cell_type": "code", - "outputs": [], - "execution_count": null - }, - { - "metadata": {}, - "source": [ - "## Step 3: Create a Model and Model Run" - ], - "cell_type": "markdown" - }, - { - "metadata": {}, - "source": [ - "# create Model\n", - "model = client.create_model(name=\"with_aggregated_custom_metrics\" + str(uuid.uuid4()),\n", - " ontology_id=ontology.uid)\n", - "# create Model Run\n", - "model_run = model.create_model_run(\"iteration 1\")" - ], - "cell_type": "code", - "outputs": [], - "execution_count": null - }, - { - "metadata": {}, - "source": [ - "## Step 4: Send data rows to the Model Run" - ], - "cell_type": "markdown" - }, - { - "metadata": {}, - "source": [ - "model_run.upsert_data_rows(global_keys=[data_row.global_key for data_row in dataset.export_data_rows()])" - ], - "cell_type": "code", - "outputs": [], - "execution_count": null - }, - { - "metadata": {}, - "source": [ - "## Step 5. Create the predictions payload\n", - "\n", - "Create the prediction payload using the snippets of code in ***Supported Predictions*** section.\n", - "\n", - "The resulting label_ndjson should have exactly the same content for predictions that are supported by both (with exception of the uuid strings that are generated)" - ], - "cell_type": "markdown" - }, - { - "metadata": {}, - "source": [ - "# Create a Label for predictions\n", - "label_predictions = [Label(\n", - " data=ImageData(uid=data_row.uid),\n", - " annotations = [\n", - " radio_prediction,\n", - " checklist_prediction,\n", - " bbox_prediction,\n", - " bbox_with_radio_subclass_prediction,\n", - " polygon_prediction,\n", - " # mask_prediction,\n", - " point_prediction,\n", - " text_annotation,\n", - " ]\n", - ") for data_row in dataset.export_data_rows()]\n", + ")\n", "\n", - "# Convert the prediction label from a Labelbox class object to the underlying NDJSON format required for upload - uploads can be directly built in this syntax as well\n", - "ndjson_prediction = list(NDJsonConverter.serialize(label_predictions))" + "ontology = client.create_ontology(\"Image Prediction Import Demo\",\n", + " ontology_builder.asdict(),\n", + " media_type=lb.MediaType.Image\n", + " )" ], "cell_type": "code", "outputs": [], @@ -651,134 +683,34 @@ { "metadata": {}, "source": [ - "If using NDJSON" + "## Step 3: Create a Model and Model Run" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "## Custom metrics" + "# create Model\n", + "model = client.create_model(name=\"model_with_aggregated_custom_metrics\" + str(uuid.uuid4()),\n", + " ontology_id=ontology.uid)\n", + "# create Model Run\n", + "model_run = model.create_model_run(\"iteration 1\")" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## Step 4: Send data rows to the Model Run" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "# Special case for custom feature metrics\n", - "\n", - "# NDJSON\n", - "radio_prediction_ndjson = {\n", - " 'name': 'radio_question',\n", - " 'answer': {'name': 'first_radio_answer', 'confidence': 0.5, 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.1 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ]}\n", - "}\n", - "\n", - "########## Nested Classifications are only supported with NDJSON tools ##########\n", - "\n", - "nested_radio_prediction_ndjson = {\n", - " \"name\": \"nested_radio_question\",\n", - " \"confidence\": 0.5 ,\n", - " \"answer\": { \"name\": \"first_radio_answer\", \"confidence\": 0.5, 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ] },\n", - " \"classifications\" : [\n", - " {\n", - " \"name\": \"sub_radio_question\",\n", - " \"answer\": {\"name\": \"first_sub_radio_answer\", \"confidence\": 0.5, 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ] }\n", - " }\n", - " ]\n", - "}\n", - "\n", - "\n", - "nested_checklist_prediction_ndjson = {\n", - " \"name\": \"nested_checklist_question\",\n", - " \"confidence\": 0.5 ,\n", - " \"answer\": [{\n", - " \"name\": \"first_checklist_answer\",\n", - " \"confidence\": 0.5,\n", - " 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ],\n", - " \"classifications\" : [\n", - " {\n", - " \"name\": \"sub_checklist_question\",\n", - " \"answer\": {\"name\": \"first_sub_checklist_answer\", \"confidence\": 0.5, 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ] }\n", - " }\n", - " ]\n", - " }]\n", - "}\n", - "\n", - "############ Checklist ############\n", - "checklist_prediction_ndjson = {\n", - " 'name': 'checklist_question',\n", - " 'answer': [\n", - " {'name': 'first_checklist_answer' , 'confidence': 0.5, 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ]},\n", - " {'name': 'second_checklist_answer', 'confidence': 0.5, 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ]}\n", - " ]\n", - "}\n", - "all_annotations = [\n", - " radio_prediction_ndjson,\n", - " nested_radio_prediction_ndjson,\n", - " checklist_prediction_ndjson,\n", - " nested_checklist_prediction_ndjson,\n", - " ]\n", - "\n", - "three_k_annotations = all_annotations\n", - "\n", - "ndjson_prediction_method2 = []\n", - "dr_ids = [data_row.uid for data_row in dataset.export_data_rows()]\n", - "for data_row_id in dr_ids:\n", - " for annot in three_k_annotations:\n", - " ndjson_prediction_method2.append({\n", - " **annot,\n", - " 'uuid': str(uuid.uuid4()),\n", - " 'dataRow': {'id': data_row_id },\n", - " })" + "model_run.upsert_data_rows(global_keys=[global_key])" ], "cell_type": "code", "outputs": [], @@ -787,100 +719,44 @@ { "metadata": {}, "source": [ - "## OCR special case" + "## Step 5. Create the predictions payload\n", + "\n", + "Create the prediction payload using the snippets of code in ***Supported Predictions*** section.\n", + "\n", + "The resulting label_ndjson should have exactly the same content for predictions that are supported by both (with exception of the uuid strings that are generated)" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ - "# Special case for OCR\n", - "\n", - "bbox_with_free_text_subclass_prediction_ndjson1 = {\n", - " \"name\": \"bbox_with_free_text_subclassification\",\n", - " \"confidence\": 0.5,\n", - " \"classifications\": [{\n", - " \"name\": \"sub_text_classification\",\n", - " \"confidence\": 0.5,\n", - " 'answer': 'aaaaaaaa - matching nicely',\n", - "\n", - " }],\n", - " \"bbox\": {\n", - " \"top\": 933,\n", - " \"left\": 541,\n", - " \"height\": 191,\n", - " \"width\": 330\n", - " }\n", - "}\n", - "bbox_with_free_text_subclass_prediction_ndjson2 = {\n", - " \"name\": \"bbox_with_free_text_subclassification\",\n", - " \"confidence\": 0.5,\n", - " \"classifications\": [{\n", - " \"name\": \"sub_text_classification\",\n", - " \"confidence\": 0.5,\n", - " 'answer': 'zzzzzzz',\n", - "\n", - " }],\n", - " \"bbox\": {\n", - " \"top\": 913,\n", - " \"left\": 521,\n", - " \"height\": 50,\n", - " \"width\": 50\n", - " }\n", - "}\n", - "all_annotations = [\n", - " bbox_with_free_text_subclass_prediction_ndjson1,\n", - " bbox_with_free_text_subclass_prediction_ndjson2,\n", - " ]\n", - "\n", - "three_k_annotations = []\n", - "three_k_annotations.extend(all_annotations)\n", - "\n", - "ndjson_prediction_method2 = []\n", - "dr_ids = [data_row.uid for data_row in dataset.export_data_rows()]\n", - "for data_row_id in dr_ids:\n", - " for annot in three_k_annotations:\n", - " ndjson_prediction_method2.append({\n", - " **annot,\n", - " 'uuid': str(uuid.uuid4()),\n", - " 'dataRow': {'id': data_row_id },\n", - " })" + "If using NDJSON" ], - "cell_type": "code", - "outputs": [], - "execution_count": null + "cell_type": "markdown" }, { "metadata": {}, "source": [ - "\n", - "all_annotations = [\n", - " radio_prediction_ndjson,\n", - " checklist_prediction_ndjson,\n", - " bbox_prediction_ndjson,\n", - " bbox_with_radio_subclass_prediction_ndjson,\n", - " polygon_prediction_ndjson,\n", - " # mask_prediction_ndjson,\n", - " point_prediction_ndjson,\n", - " polyline_prediction_ndjson,\n", - " text_annotation_ndjson,\n", - " nested_radio_prediction_ndjson,\n", - " nested_checklist_prediction_ndjson\n", - " ]\n", - "\n", - "# three_k_annotations = bbox_predictions\n", - "# three_k_annotations.extend(all_annotations)\n", - "three_k_annotations = all_annotations\n", - "\n", - "ndjson_prediction_method2 = []\n", - "dr_ids = [data_row.uid for data_row in dataset.export_data_rows()]\n", - "for data_row_id in dr_ids:\n", - " for annot in three_k_annotations:\n", - " ndjson_prediction_method2.append({\n", - " **annot,\n", - " 'uuid': str(uuid.uuid4()),\n", - " 'dataRow': {'id': data_row_id }\n", - " })" + "label_prediction_ndjson = []\n", + "\n", + "for annot in [\n", + " radio_prediction_ndjson,\n", + " checklist_prediction_ndjson,\n", + " bbox_prediction_ndjson,\n", + " bbox_with_radio_subclass_prediction_ndjson,\n", + " polygon_prediction_ndjson,\n", + " mask_prediction_ndjson,\n", + " mask_with_text_subclass_prediction_ndjson,\n", + " point_prediction_ndjson,\n", + " polyline_prediction_ndjson,\n", + " text_annotation_ndjson,\n", + " nested_radio_prediction_ndjson,\n", + " nested_checklist_prediction_ndjson\n", + "]:\n", + " annot.update({\n", + " \"dataRow\": {\"globalKey\": global_key}\n", + " })\n", + " label_prediction_ndjson.append(annot)" ], "cell_type": "code", "outputs": [], @@ -899,7 +775,7 @@ "# Upload the prediction label to the Model Run\n", "upload_job_prediction = model_run.add_predictions(\n", " name=\"prediction_upload_job\"+str(uuid.uuid4()),\n", - " predictions=ndjson_prediction_method2)\n", + " predictions=label_prediction_ndjson)\n", "\n", "# Errors will appear for prediction uploads that failed.\n", "print(\"Errors:\", upload_job_prediction.errors)" @@ -929,8 +805,7 @@ "source": [ "# Create a Labelbox project\n", "project = client.create_project(name=\"image_prediction_many_kinds\",\n", - " quality_mode=QualityMode.Benchmark,\n", - " media_type=MediaType.Image)\n", + " media_type=lb.MediaType.Image)\n", "project.setup_editor(ontology)" ], "cell_type": "code", @@ -948,9 +823,9 @@ "metadata": {}, "source": [ "project.create_batch(\n", - " \"batch_predictions_demo1\", # Each batch in a project must have a unique name\n", - " [data_row.uid for data_row in dataset.export_data_rows()], # A list of data rows or data row ids\n", - " 5 # priority between 1(Highest) - 5(lowest)\n", + " \"batch_predictions_demo\", # Each batch in a project must have a unique name\n", + " global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys\n", + " priority=5 # priority between 1(Highest) - 5(lowest)\n", ")" ], "cell_type": "code", @@ -968,140 +843,133 @@ "metadata": {}, "source": [ "########### Annotations ###########\n", - "radio_annotation_ndjson = {\n", - " \"name\": \"radio_question\",\n", - " \"answer\": {\"name\": \"first_radio_answer\"}\n", - "}\n", - "\n", - "nested_radio_annotation_ndjson = {\n", - " \"name\": \"nested_radio_question\",\n", - " \"answer\": {\"name\": \"first_radio_answer\"},\n", - " \"classifications\" : [\n", - " {\"name\": \"sub_radio_question\", \"answer\": {\"name\": \"first_sub_radio_answer\"}}\n", - " ]\n", - "}\n", + "radio_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"radio_question\",\n", + " value=lb_types.Radio(answer=lb_types.ClassificationAnswer(\n", + " name=\"second_radio_answer\"))\n", + ")\n", "\n", - "checklist_annotation_ndjson = {\n", - " \"name\": \"checklist_question\",\n", - " \"answer\": [\n", - " {\"name\": \"first_checklist_answer\"},\n", - " {\"name\": \"second_checklist_answer\"}\n", - " ]\n", - "}\n", + "nested_radio_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"nested_radio_question\",\n", + " value=lb_types.Radio(\n", + " answer=lb_types.ClassificationAnswer(\n", + " name=\"first_radio_answer\",\n", + " classifications=[\n", + " lb_types.ClassificationAnnotation(\n", + " name=\"sub_radio_question\",\n", + " value=lb_types.Radio(\n", + " answer=lb_types.ClassificationAnswer(\n", + " name=\"first_sub_radio_answer\"\n", + " )\n", + " )\n", + " )\n", + " ]\n", + " )\n", + " )\n", + ")\n", "\n", - "bbox_annotation_ndjson = {\n", - " \"name\": \"bounding_box\",\n", - " \"bbox\": {\n", - " \"top\": 877,\n", - " \"left\": 1490,\n", - " \"height\": 130,\n", - " \"width\": 125\n", - " }\n", - "}\n", + "nested_checklist_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"nested_checklist_question\",\n", + " value=lb_types.Checklist(\n", + " answer=[lb_types.ClassificationAnswer(\n", + " name=\"first_checklist_answer\",\n", + " classifications=[\n", + " lb_types.ClassificationAnnotation(\n", + " name=\"sub_checklist_question\",\n", + " value=lb_types.Checklist(\n", + " answer=[lb_types.ClassificationAnswer(\n", + " name=\"first_sub_checklist_answer\"\n", + " )]\n", + " ))\n", + " ]\n", + " )]\n", + " )\n", + ")\n", "\n", - "bbox_with_radio_subclass_ndjson = {\n", - " \"name\": \"bbox_with_radio_subclass\",\n", - " \"classifications\": [{\n", - " \"name\": \"sub_radio_question\",\n", - " \"answer\":\n", - " { \"name\":\"first_sub_radio_answer\" }\n", + "checklist_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"checklist_question\",\n", + " value=lb_types.Checklist(answer=[\n", + " lb_types.ClassificationAnswer(name=\"first_checklist_answer\"),\n", + " lb_types.ClassificationAnswer(name=\"second_checklist_answer\")\n", + " ])\n", + ")\n", "\n", - " }],\n", - " \"bbox\": {\n", - " \"top\": 533,\n", - " \"left\": 841,\n", - " \"height\": 191,\n", - " \"width\": 230\n", - " }\n", - "}\n", + "bbox_annotation = lb_types.ObjectAnnotation(\n", + " name = \"bounding_box\",\n", + " value=lb_types.Rectangle(\n", + " start=lb_types.Point(x=1690, y=977), # x = left, y = top\n", + " end=lb_types.Point(x=1915, y=1307), # x= left + width , y = top + height\n", + " )\n", + ")\n", "\n", - "polygon_annotation_ndjson = {\n", - " \"name\": \"polygon\",\n", - " \"polygon\": [\n", - " {\"x\": 1489.581, \"y\": 183.934},\n", - " {\"x\": 2278.306, \"y\": 256.885},\n", - " {\"x\": 2428.197, \"y\": 200.437},\n", - " {\"x\": 2560.0, \"y\": 335.419},\n", - " {\"x\": 2557.386, \"y\": 503.165},\n", - " {\"x\": 2320.596, \"y\": 503.103},\n", - " {\"x\": 2156.083, \"y\": 628.943},\n", - " {\"x\": 2161.111, \"y\": 785.519},\n", - " {\"x\": 2002.115, \"y\": 894.647},\n", - " {\"x\": 1838.456, \"y\": 877.874},\n", - " {\"x\": 1436.53, \"y\": 874.636},\n", - " {\"x\": 1411.403, \"y\": 758.579},\n", - " {\"x\": 1353.853, \"y\": 751.74},\n", - " {\"x\": 1345.264, \"y\": 453.461},\n", - " {\"x\": 1426.011, \"y\": 421.129},\n", - " {\"x\": 1489.581, \"y\": 183.934}\n", + "bbox_with_radio_subclass_annotation = lb_types.ObjectAnnotation(\n", + " name=\"bbox_with_radio_subclass\",\n", + " value=lb_types.Rectangle(\n", + " start=lb_types.Point(x=541, y=933), # x = left, y = top\n", + " end=lb_types.Point(x=871, y=1124), # x= left + width , y = top + height\n", + " ),\n", + " classifications=[\n", + " \tlb_types.ClassificationAnnotation(\n", + " \tname=\"sub_radio_question\",\n", + " \t\tvalue=lb_types.Radio(answer=lb_types.ClassificationAnswer(name=\"first_sub_radio_answer\", confidence=0.5))\n", + " )\n", " ]\n", - "}\n", + ")\n", "\n", - "mask_annotation_ndjson = {\n", - " \"name\": \"mask\",\n", - " \"classifications\": [],\n", - " \"mask\": {\"instanceURI\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png\",\n", - " \"colorRGB\": (0, 0, 0)}\n", - "}\n", + "polygon_annotation = lb_types.ObjectAnnotation(\n", + " name = \"polygon\",\n", + " value=lb_types.Polygon(\n", + " points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419),\n", + " lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519),\n", + " lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579),\n", + " lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)]\n", + " ),\n", + ")\n", "\n", + "text_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"free_text\",\n", + " value=lb_types.Text(answer=\"sample text\")\n", + ")\n", "\n", - "point_annotation_ndjson = {\n", - " \"name\": \"point\",\n", - " \"classifications\": [],\n", - " \"point\": {\"x\": 1166.606, \"y\": 1441.768}\n", - "}\n", + "response = requests.get(url)\n", "\n", - "point_annotation_ndjson = {\n", - " \"name\": \"point\",\n", - " \"classifications\": [],\n", - " \"point\": {\"x\": 1166.606, \"y\": 1441.768}\n", - "}\n", + "mask_data = lb.types.MaskData(im_bytes=response.content)\n", "\n", - "polyline_annotation_ndjson = {\n", - " \"name\": \"polyline\",\n", - " \"classifications\": [],\n", - " \"line\": [\n", - " {\"x\": 2534.353, \"y\": 249.471},\n", - " {\"x\": 2429.492, \"y\": 182.092},\n", - " {\"x\": 2294.322, \"y\": 221.962},\n", - " {\"x\": 2224.491, \"y\": 180.463},\n", - " {\"x\": 2136.123, \"y\": 204.716},\n", - " {\"x\": 1712.247, \"y\": 173.949},\n", - " {\"x\": 1703.838, \"y\": 84.438},\n", - " {\"x\": 1579.772, \"y\": 82.61},\n", - " {\"x\": 1583.442, \"y\": 167.552},\n", - " {\"x\": 1478.869, \"y\": 164.903},\n", - " {\"x\": 1418.941, \"y\": 318.149},\n", - " {\"x\": 1243.128, \"y\": 400.815},\n", - " {\"x\": 1022.067, \"y\": 319.007},\n", - " {\"x\": 892.367, \"y\": 379.216},\n", - " {\"x\": 670.273, \"y\": 364.408},\n", - " {\"x\": 613.114, \"y\": 288.16},\n", - " {\"x\": 377.559, \"y\": 238.251},\n", - " {\"x\": 368.087, \"y\": 185.064},\n", - " {\"x\": 246.557, \"y\": 167.286},\n", - " {\"x\": 236.648, \"y\": 285.61},\n", - " {\"x\": 90.929, \"y\": 326.412}\n", - " ]\n", - "}\n", + "mask_annotation = lb_types.ObjectAnnotation(\n", + " name=\"mask\",\n", + " value=lb_types.Mask(\n", + " mask=mask_data,\n", + " color=(255, 255, 255))\n", + ")\n", + "response_2 = requests.get(url_2)\n", + "mask_data_2 = lb_types.MaskData(im_bytes=response_2.content)\n", + "mask_with_text_subclass_annotation = lb_types.ObjectAnnotation(\n", + " name = \"mask_with_text_subclass\", # must match your ontology feature\"s name\n", + " value=lb_types.Mask(\n", + " mask=mask_data_2,\n", + " color=(255, 255, 255)),\n", + " classifications=[\n", + " lb_types.ClassificationAnnotation(\n", + " name=\"sub_free_text\",\n", + " value=lb_types.Text(answer=\"free text answer\")\n", + " )]\n", + ")\n", "\n", - "nested_checklist_annotation_ndjson = {\n", - " \"name\": \"nested_checklist_question\",\n", - " \"answer\": [{\n", - " \"name\": \"first_checklist_answer\",\n", - " \"classifications\" : [\n", - " {\n", - " \"name\": \"sub_checklist_question\",\n", - " \"answer\": {\"name\": \"first_sub_checklist_answer\"}\n", - " }\n", - " ]\n", - " }]\n", - "}\n", + "point_annotation = lb_types.ObjectAnnotation(\n", + " name = \"point\",\n", + " value = lb_types.Point(x=1166.606, y=1441.768),\n", + ")\n", "\n", - "text_annotation_ndjson = {\n", - " \"name\": \"free_text\",\n", - " \"answer\": \"sample text\",\n", - "}\n" + "polyline_annotation = lb_types.ObjectAnnotation(\n", + " name = \"polyline\",\n", + " value=lb_types.Line(\n", + " points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716),\n", + " lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552),\n", + " lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007),\n", + " lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251),\n", + " lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)]\n", + " )\n", + ")" ], "cell_type": "code", "outputs": [], @@ -1118,30 +986,25 @@ "metadata": {}, "source": [ "# Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations\n", + "label = []\n", "annotations = [\n", - " radio_annotation_ndjson,\n", - " checklist_annotation_ndjson,\n", - " bbox_annotation_ndjson,\n", - " bbox_with_radio_subclass_ndjson,\n", - " polygon_annotation_ndjson,\n", - " mask_annotation_ndjson,\n", - " point_annotation_ndjson,\n", - " polyline_annotation_ndjson,\n", - " nested_radio_annotation_ndjson,\n", - " nested_checklist_annotation_ndjson,\n", - " text_annotation_ndjson\n", - " ]\n", - "# gts = bbox_annotations\n", - "gts = (annotations)\n", - "ndjson_annotation = []\n", - "for data_row in dataset.export_data_rows():\n", - " for annot in gts:\n", - " ndjson_annotation.append({\n", - " **annot,\n", - " 'uuid': str(uuid.uuid4()),\n", - " 'dataRow': {'id': data_row.uid}\n", - " })\n", - "\n" + " radio_annotation,\n", + " nested_radio_annotation,\n", + " checklist_annotation,\n", + " nested_checklist_annotation,\n", + " text_annotation,\n", + " bbox_annotation,\n", + " bbox_with_radio_subclass_annotation,\n", + " polygon_annotation,\n", + " mask_annotation,\n", + " mask_with_text_subclass_annotation,\n", + " point_annotation,\n", + " polyline_annotation\n", + "]\n", + "label.append(\n", + " lb_types.Label(\n", + " data=lb_types.ImageData(global_key=global_key),\n", + " annotations=annotations))" ], "cell_type": "code", "outputs": [], @@ -1157,15 +1020,16 @@ { "metadata": {}, "source": [ - "upload_job_annotation = LabelImport.create_from_objects(\n", + "upload_job_annotation = lb.LabelImport.create_from_objects(\n", " client = client,\n", " project_id = project.uid,\n", " name=\"annotation_import_\" + str(uuid.uuid4()),\n", - " labels=ndjson_annotation)\n", + " labels=label)\n", "\n", "upload_job_annotation.wait_until_done()\n", "# Errors will appear for annotation uploads that failed.\n", - "print(\"Errors:\", upload_job_annotation.errors)\n" + "print(\"Errors:\", upload_job_annotation.errors)\n", + "print(\"Status of uploads: \", upload_job_annotation.statuses)" ], "cell_type": "code", "outputs": [], @@ -1181,11 +1045,7 @@ { "metadata": {}, "source": [ - "# get the labels id from the project\n", - "task = project.export_v2()\n", - "task.wait_till_done()\n", - "label_ids = [l[\"id\"] for dr in task.result for l in dr[\"projects\"][project.uid][\"labels\"]]\n", - "model_run.upsert_labels(label_ids)" + "model_run.upsert_labels(project_id=project.uid)" ], "cell_type": "code", "outputs": [], diff --git a/examples/project_configuration/webhooks.ipynb b/examples/project_configuration/webhooks.ipynb index 904e13f91..ec54de860 100644 --- a/examples/project_configuration/webhooks.ipynb +++ b/examples/project_configuration/webhooks.ipynb @@ -1,16 +1,18 @@ { + "nbformat": 4, + "nbformat_minor": 5, + "metadata": {}, "cells": [ { - "cell_type": "markdown", "metadata": {}, "source": [ "\n", " \n", "" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -22,32 +24,28 @@ "\n", "" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "# Webhook Configuration" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "id": "18fc16be", "metadata": {}, "source": [ "Webhooks are supported for the following events:\n", "* label_created\n", "* label_updated\n", "* label_deleted" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, - "id": "ad5486cd", "metadata": {}, - "outputs": [], "source": [ "!pip install labelbox\n", "!pip install requests\n", @@ -55,14 +53,13 @@ "!pip install hashlib\n", "!pip install flask\n", "!pip install Werkzeug" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "a2a8f3b2", "metadata": {}, - "outputs": [], "source": [ "import labelbox as lb\n", "from flask import Flask, request\n", @@ -75,14 +72,13 @@ "import os\n", "from getpass import getpass\n", "import socket" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "99a23b2b", "metadata": {}, - "outputs": [], "source": [ "# If you don't want to give google access to drive you can skip this cell\n", "# and manually set `API_KEY` below.\n", @@ -99,46 +95,44 @@ " API_KEY = getpass(\"Please enter your labelbox api key\")\n", " if COLAB:\n", " envvar_handler.add_env(\"LABELBOX_API_KEY\", API_KEY)" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "ee93a4ab", "metadata": {}, - "outputs": [], "source": [ "# Set this to a project that you want to use for the webhook\n", "PROJECT_ID = \"\"\n", "# Only update this if you have an on-prem deployment\n", "ENDPOINT = \"https://api.labelbox.com/graphql\"" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "a161946b", "metadata": {}, - "outputs": [], "source": [ "client = lb.Client(api_key=API_KEY, endpoint=ENDPOINT)" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "29890bf8", "metadata": {}, - "outputs": [], "source": [ "# We are using port 3001 for this example.\n", "# Feel free to set to whatever port you want\n", "WH_PORT = 3001" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "markdown", - "id": "8e48bb5f", "metadata": {}, "source": [ "### Configure NGROK (Optional)\n", @@ -149,44 +143,38 @@ "2. Download ngrok and extract the zip file\n", "3. Add ngrok to your path\n", "4. Add the authtoken `ngrok authtoken `" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, - "id": "005702a2", "metadata": {}, - "outputs": [], "source": [ "if not COLAB:\n", " os.system(f\"ngrok http {WH_PORT} &\")" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "markdown", - "id": "4417e81b", "metadata": {}, "source": [ "### Configure server to receive requests" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, - "id": "4bbfd57c", "metadata": {}, - "outputs": [], "source": [ "# This can be any secret that matches your webhook config (we will set later)\n", "secret = b\"example_secret\"" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "5b98f952", "metadata": {}, - "outputs": [], "source": [ "app = Flask(__name__)\n", "\n", @@ -217,49 +205,44 @@ "\n", "thread = threading.Thread(target=lambda: run_simple(\"0.0.0.0\", WH_PORT, app))\n", "thread.start()" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "markdown", - "id": "2a6fce6d", "metadata": {}, "source": [ "#### Test server" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, - "id": "c53f862c", "metadata": {}, - "outputs": [], "source": [ "print(requests.get(\"http://localhost:3001\").text)" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "markdown", - "id": "b84254a8", "metadata": {}, "source": [ "### Create Webhook" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "id": "916f3436", "metadata": {}, "source": [ "- Set ip address if your ip is publicly accessible.\n", "- Otherwise use the following to get ngrok public_url" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, - "id": "41832282", "metadata": {}, - "outputs": [], "source": [ "if not COLAB:\n", " res = requests.get(\"http://localhost:4040/api/tunnels\")\n", @@ -274,14 +257,13 @@ "else:\n", " public_url = f\"http://{socket.gethostbyname(socket.getfqdn(socket.gethostname()))}\"\n", "print(public_url)" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "2f18123e", "metadata": {}, - "outputs": [], "source": [ "# Set project to limit the scope to a single project\n", "project = client.get_project(PROJECT_ID)\n", @@ -292,34 +274,31 @@ " url=public_url,\n", " secret=secret.decode(),\n", " project=project)" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, - "id": "b9226f8f", "metadata": {}, - "outputs": [], "source": [ "# Ok so we should be configured assuming everything is setup correctly.\n", "# Go to the following url and make a new label to see if it works\n", "print(f\"https://app.labelbox.com/projects/{PROJECT_ID}\")" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "markdown", - "id": "bae73fb4", "metadata": {}, "source": [ "### Update Webhook" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, - "id": "d7070ab0", "metadata": {}, - "outputs": [], "source": [ "# url, topics, and status can all be updated\n", "updated_url = f\"{public_url}/webhook-endpoint\"\n", @@ -328,20 +307,20 @@ "# Go to the following url and try one last time.\n", "# Any supported action should work (create, delete, or update a label)\n", "print(f\"https://app.labelbox.com/projects/{PROJECT_ID}\")" - ] + ], + "cell_type": "code", + "outputs": [], + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "### List and delete all webhooks" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, - "outputs": [], "source": [ "# DELETE:\n", "webhook.update(status=lb.Webhook.Status.INACTIVE.value)\n", @@ -356,14 +335,10 @@ "# for webhook in webhooks:\n", "# print(webhook)\n", "# webhook.update(status = lb.Webhook.Status.INACTIVE.value)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" + ], + "cell_type": "code", + "outputs": [], + "execution_count": null } - }, - "nbformat": 4, - "nbformat_minor": 5 -} + ] +} \ No newline at end of file From a88490e18cd523a87b7d4130562df53f312d7253 Mon Sep 17 00:00:00 2001 From: ovalle15 Date: Tue, 27 Feb 2024 19:13:41 -0500 Subject: [PATCH 3/4] AO updates removed masks since they are not supported --- .../custom_metrics_demo.ipynb | 110 ------------------ 1 file changed, 110 deletions(-) diff --git a/examples/model_diagnostics/custom_metrics_demo.ipynb b/examples/model_diagnostics/custom_metrics_demo.ipynb index b52eaaa23..97b9774ff 100644 --- a/examples/model_diagnostics/custom_metrics_demo.ipynb +++ b/examples/model_diagnostics/custom_metrics_demo.ipynb @@ -82,7 +82,6 @@ "metadata": {}, "source": [ "API_KEY = \"\"\n", - "\n", "client = lb.Client(API_KEY)" ], "cell_type": "code", @@ -384,75 +383,6 @@ "outputs": [], "execution_count": null }, - { - "metadata": {}, - "source": [ - "### Segmentation mask" - ], - "cell_type": "markdown" - }, - { - "metadata": {}, - "source": [ - "### Raster Segmentation (Byte string array)\n", - "url = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png\"\n", - "\n", - "mask_prediction_ndjson = {\n", - " \"name\": \"mask\",\n", - " \"confidence\": 0.5,\n", - " 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ],\n", - " \"classifications\": [],\n", - " \"mask\": {\"instanceURI\": url,\n", - " \"colorRGB\": (255, 255, 255)}\n", - "}\n", - "\n" - ], - "cell_type": "code", - "outputs": [], - "execution_count": null - }, - { - "metadata": {}, - "source": [ - "### Segmentation mask with nested classification" - ], - "cell_type": "markdown" - }, - { - "metadata": {}, - "source": [ - "url_2 = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg_with_subclass.png\"\n", - "\n", - "# NDJSON using instanceURI, bytes array is not fully supported.\n", - "mask_with_text_subclass_prediction_ndjson = {\n", - " \"name\": \"mask_with_text_subclass\",\n", - " 'customMetrics': [\n", - " { 'name': 'iou', 'value': 0.5 },\n", - " { 'name': 'f1', 'value': 0.33 },\n", - " { 'name': 'precision', 'value': 0.55 },\n", - " { 'name': 'recall', 'value': 0.33 },\n", - " { 'name': 'tagsCount', 'value': 43 },\n", - " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n", - " ],\n", - " \"mask\": {\"instanceURI\": url_2,\n", - " \"colorRGB\": (255, 255, 255)},\n", - " \"classifications\":[{\n", - " \"name\": \"sub_free_text\",\n", - " \"answer\": \"free text answer\"\n", - " }]\n", - "}" - ], - "cell_type": "code", - "outputs": [], - "execution_count": null - }, { "metadata": {}, "source": [ @@ -651,18 +581,6 @@ " lb.Tool(\n", " tool=lb.Tool.Type.POLYGON,\n", " name=\"polygon\"),\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.RASTER_SEGMENTATION,\n", - " name=\"mask\"),\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.RASTER_SEGMENTATION,\n", - " name=\"mask_with_text_subclass\",\n", - " classifications=[\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.TEXT,\n", - " name=\"sub_free_text\")\n", - " ]\n", - " ),\n", " \t lb.Tool(\n", " tool=lb.Tool.Type.POINT,\n", " name=\"point\"),\n", @@ -745,8 +663,6 @@ " bbox_prediction_ndjson,\n", " bbox_with_radio_subclass_prediction_ndjson,\n", " polygon_prediction_ndjson,\n", - " mask_prediction_ndjson,\n", - " mask_with_text_subclass_prediction_ndjson,\n", " point_prediction_ndjson,\n", " polyline_prediction_ndjson,\n", " text_annotation_ndjson,\n", @@ -931,30 +847,6 @@ " value=lb_types.Text(answer=\"sample text\")\n", ")\n", "\n", - "response = requests.get(url)\n", - "\n", - "mask_data = lb.types.MaskData(im_bytes=response.content)\n", - "\n", - "mask_annotation = lb_types.ObjectAnnotation(\n", - " name=\"mask\",\n", - " value=lb_types.Mask(\n", - " mask=mask_data,\n", - " color=(255, 255, 255))\n", - ")\n", - "response_2 = requests.get(url_2)\n", - "mask_data_2 = lb_types.MaskData(im_bytes=response_2.content)\n", - "mask_with_text_subclass_annotation = lb_types.ObjectAnnotation(\n", - " name = \"mask_with_text_subclass\", # must match your ontology feature\"s name\n", - " value=lb_types.Mask(\n", - " mask=mask_data_2,\n", - " color=(255, 255, 255)),\n", - " classifications=[\n", - " lb_types.ClassificationAnnotation(\n", - " name=\"sub_free_text\",\n", - " value=lb_types.Text(answer=\"free text answer\")\n", - " )]\n", - ")\n", - "\n", "point_annotation = lb_types.ObjectAnnotation(\n", " name = \"point\",\n", " value = lb_types.Point(x=1166.606, y=1441.768),\n", @@ -996,8 +888,6 @@ " bbox_annotation,\n", " bbox_with_radio_subclass_annotation,\n", " polygon_annotation,\n", - " mask_annotation,\n", - " mask_with_text_subclass_annotation,\n", " point_annotation,\n", " polyline_annotation\n", "]\n", From 40c4cbef7647d287fc1dbec843201d9dbf04a9bf Mon Sep 17 00:00:00 2001 From: ovalle15 Date: Tue, 27 Feb 2024 19:18:06 -0500 Subject: [PATCH 4/4] Updated open in colab and github links to point to master --- examples/model_diagnostics/custom_metrics_demo.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/model_diagnostics/custom_metrics_demo.ipynb b/examples/model_diagnostics/custom_metrics_demo.ipynb index 97b9774ff..42b8575d1 100644 --- a/examples/model_diagnostics/custom_metrics_demo.ipynb +++ b/examples/model_diagnostics/custom_metrics_demo.ipynb @@ -17,12 +17,12 @@ "source": [ "\n", "\n", - "\n", "\n", "\n", "\n", - "\n", "" ],