diff --git a/examples/model_diagnostics/custom_metrics_demo.ipynb b/examples/model_diagnostics/custom_metrics_demo.ipynb
index a4c73a581..42b8575d1 100644
--- a/examples/model_diagnostics/custom_metrics_demo.ipynb
+++ b/examples/model_diagnostics/custom_metrics_demo.ipynb
@@ -1,6 +1,6 @@
{
"nbformat": 4,
- "nbformat_minor": 5,
+ "nbformat_minor": 0,
"metadata": {},
"cells": [
{
@@ -15,28 +15,19 @@
{
"metadata": {},
"source": [
+ "\n",
"| \n",
"\n",
" | \n",
"\n",
"\n",
- "\n",
" | "
],
"cell_type": "markdown"
},
- {
- "metadata": {},
- "source": [
- " \n",
- "\n",
- " ![](\"https://labelbox.com/blog/content/images/2021/02/logo-v4.svg\") \n",
- " | "
- ],
- "cell_type": "markdown"
- },
{
"metadata": {},
"source": [
@@ -47,28 +38,21 @@
"* Measuring model quality is critical to efficiently building models. It is important that the metrics used to measure model quality closely align with the business objectives for the model. Otherwise, slight changes in model quality, as they related to these core objectives, are lost to noise. Custom metrics enables users to measure model quality in terms of their exact business goals. By incorporating custom metrics into workflows, users can:\n",
" * Iterate faster\n",
" * Measure and report on model quality\n",
- " * Understand marginal value of additional labels and modeling efforts\n",
- "\n",
- " \n",
- "* Checkout this [notebook](custom_metrics_basics.ipynb) for more details on the metrics types and built in functions for calculating metrics.\n"
+ " * Understand marginal value of additional labels and modeling efforts\n"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "## Environment Setup\n",
- "\n",
- "Install dependencies"
+ "## Environment setup"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "!pip install -q \"labelbox[data]\" \\\n",
- " scikit-image \\\n",
- " tensorflow"
+ "!pip install -q 'labelbox[data]'"
],
"cell_type": "code",
"outputs": [],
@@ -77,13 +61,10 @@
{
"metadata": {},
"source": [
- "# Run these if running in a colab notebook\n",
- "COLAB = \"google.colab\" in str(get_ipython())\n",
- "\n",
- "if COLAB:\n",
- " !git clone https://github.com/Labelbox/labelbox-python.git\n",
- " !cd labelbox-python\n",
- " !mv labelbox-python/examples/model_assisted_labeling/*.py ."
+ "import uuid\n",
+ "import requests\n",
+ "import labelbox as lb\n",
+ "import labelbox.types as lb_types"
],
"cell_type": "code",
"outputs": [],
@@ -92,39 +73,89 @@
{
"metadata": {},
"source": [
- "Import libraries"
+ "## Replace with your API Key\n",
+ "Guides on [Create an API key](https://docs.labelbox.com/docs/create-an-api-key)"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "API_KEY = \"\"\n",
+ "client = lb.Client(API_KEY)"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
+ },
+ {
+ "metadata": {},
+ "source": [
+ "## Supported Predictions"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "### Classification: Radio (single-choice)"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "import sys\n",
- "\n",
- "sys.path.append('../model_assisted_labeling')\n",
- "\n",
- "import uuid\n",
- "import numpy as np\n",
- "from skimage import measure\n",
- "import requests\n",
- "from tqdm import notebook\n",
- "import requests\n",
- "import csv\n",
- "import os\n",
- "\n",
- "import labelbox as lb\n",
- "import labelbox.types as lb_types\n",
- "from labelbox.data.metrics.group import get_label_pairs\n",
- "from labelbox.data.metrics import (feature_miou_metric, confusion_matrix_metric,\n",
- " feature_confusion_matrix_metric)\n",
"\n",
- "try:\n",
- " from image_model import predict, load_model\n",
- "except ModuleNotFoundError:\n",
- " # !git clone https://github.com/Labelbox/labelbox-python.git\n",
- " # !cd labelbox-python && git checkout mea-dev\n",
- " # !mv labelbox-python/examples/model_assisted_labeling/*.py .\n",
- " raise Exception(\"You will need to run from the labelbox-python git repo\")"
+ "# NDJSON\n",
+ "radio_prediction_ndjson = {\n",
+ " \"name\": \"radio_question\",\n",
+ " \"answer\": {'name': 'first_radio_answer', 'confidence': 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.1 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " }\n",
+ "}"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
+ },
+ {
+ "metadata": {},
+ "source": [
+ "### Classification: checklist (multi-choice)"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "checklist_prediction_ndjson = {\n",
+ " 'name': 'checklist_question',\n",
+ " 'answer': [\n",
+ " {'name': 'first_checklist_answer' , 'confidence': 0.5, 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]},\n",
+ " {'name': 'second_checklist_answer', 'confidence': 0.5, 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]}\n",
+ " ]\n",
+ "}"
],
"cell_type": "code",
"outputs": [],
@@ -133,16 +164,73 @@
{
"metadata": {},
"source": [
- "Configure client"
+ "### Classification: Nested radio and checklist"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "PROJECT_NAME = \"Diagnostics Demo Custom Metrics\"\n",
- "MODEL_NAME = \"MSCOCO Mapillary Custom Metrics\"\n",
- "MODEL_VERSION = \"0.0.0\""
+ "nested_radio_prediction_ndjson = {\n",
+ " \"name\": \"nested_radio_question\",\n",
+ " \"confidence\": 0.5 ,\n",
+ " \"answer\": { \"name\": \"first_radio_answer\", \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " },\n",
+ " \"classifications\" : [\n",
+ " {\n",
+ " \"name\": \"sub_radio_question\",\n",
+ " \"answer\": { \"name\": \"first_sub_radio_answer\", \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " }\n",
+ " }\n",
+ " ]\n",
+ "}\n",
+ "\n",
+ "nested_checklist_prediction_ndjson = {\n",
+ " \"name\": \"nested_checklist_question\",\n",
+ " \"confidence\": 0.5 ,\n",
+ " \"answer\": [{\n",
+ " \"name\": \"first_checklist_answer\",\n",
+ " \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"classifications\" : [{\n",
+ " \"name\": \"sub_checklist_question\",\n",
+ " \"answer\": {\n",
+ " \"name\": \"first_sub_checklist_answer\", \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]}\n",
+ " }\n",
+ " ]\n",
+ " }]\n",
+ "}"
],
"cell_type": "code",
"outputs": [],
@@ -151,19 +239,72 @@
{
"metadata": {},
"source": [
- "API Key\n",
+ "### Bounding Box"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
"\n",
- "Provide a valid api key below in order to properly connect to the Labelbox Client."
+ "bbox_prediction_ndjson = {\n",
+ " \"name\": \"bounding_box\",\n",
+ " \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"bbox\": {\n",
+ " \"top\": 977,\n",
+ " \"left\": 1690,\n",
+ " \"height\": 330,\n",
+ " \"width\": 225\n",
+ " }\n",
+ "}\n"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
+ },
+ {
+ "metadata": {},
+ "source": [
+ "### Bounding box with nested classification "
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "# Add your api key\n",
- "API_KEY = None\n",
- "client = lb.Client(api_key=API_KEY)\n",
- "load_model() # initialize Tensorflow Model"
+ "## NDJSON\n",
+ "bbox_with_radio_subclass_prediction_ndjson = {\n",
+ " \"name\": \"bbox_with_radio_subclass\",\n",
+ " \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.2 },\n",
+ " { 'name': 'precision', 'value': 0.1 },\n",
+ " { 'name': 'recall', 'value': 0.3 },\n",
+ " { 'name': 'tagsCount', 'value': 23 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"classifications\": [{\n",
+ " \"name\": \"sub_radio_question\",\n",
+ " \"answer\":\n",
+ " { \"name\":\"first_sub_radio_answer\", \"confidence\": 0.5}\n",
+ "\n",
+ " }],\n",
+ " \"bbox\": {\n",
+ " \"top\": 933,\n",
+ " \"left\": 541,\n",
+ " \"height\": 191,\n",
+ " \"width\": 330\n",
+ " }\n",
+ "}"
],
"cell_type": "code",
"outputs": [],
@@ -172,63 +313,42 @@
{
"metadata": {},
"source": [
- "# Configure for whatever combination of tools and class names that you would like.\n",
- "class_mappings = {\n",
- " 1: {\n",
- " \"name\": 'person',\n",
- " \"kind\": lb.Tool.Type.POLYGON\n",
- " },\n",
- " 2: {\n",
- " \"name\": 'bicycle',\n",
- " \"kind\": lb.Tool.Type.SEGMENTATION,\n",
- " 'color': 64\n",
- " },\n",
- " 3: {\n",
- " \"name\": 'car',\n",
- " \"kind\": lb.Tool.Type.BBOX\n",
- " },\n",
- " 4: {\n",
- " \"name\": 'motorcycle',\n",
- " \"kind\": lb.Tool.Type.BBOX\n",
- " },\n",
- " 6: {\n",
- " \"name\": 'bus',\n",
- " \"kind\": lb.Tool.Type.POLYGON\n",
- " },\n",
- " 7: {\n",
- " \"name\": 'train',\n",
- " \"kind\": lb.Tool.Type.POLYGON\n",
- " },\n",
- " 8: {\n",
- " \"name\": 'truck',\n",
- " \"kind\": lb.Tool.Type.POLYGON\n",
- " },\n",
- " 10: {\n",
- " \"name\": 'traffic light',\n",
- " \"kind\": lb.Tool.Type.POINT\n",
- " },\n",
- " 11: {\n",
- " \"name\": 'fire hydrant',\n",
- " \"kind\": lb.Tool.Type.BBOX\n",
- " },\n",
- " 13: {\n",
- " \"name\": 'stop sign',\n",
- " \"kind\": lb.Tool.Type.SEGMENTATION,\n",
- " 'color': 255\n",
- " },\n",
- " 14: {\n",
- " \"name\": 'parking meter',\n",
- " \"kind\": lb.Tool.Type.POINT\n",
- " },\n",
- " 28: {\n",
- " \"name\": 'umbrella',\n",
- " \"kind\": lb.Tool.Type.SEGMENTATION,\n",
- " 'color': 128\n",
- " },\n",
- " 31: {\n",
- " \"name\": 'handbag',\n",
- " \"kind\": lb.Tool.Type.POINT\n",
- " },\n",
+ "### Polygon"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "polygon_prediction_ndjson = {\n",
+ " \"name\": \"polygon\",\n",
+ " \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"polygon\": [\n",
+ " {\"x\": 1489.581, \"y\": 183.934},\n",
+ " {\"x\": 2278.306, \"y\": 256.885},\n",
+ " {\"x\": 2428.197, \"y\": 200.437},\n",
+ " {\"x\": 2560.0, \"y\": 335.419},\n",
+ " {\"x\": 2557.386, \"y\": 503.165},\n",
+ " {\"x\": 2320.596, \"y\": 503.103},\n",
+ " {\"x\": 2156.083, \"y\": 628.943},\n",
+ " {\"x\": 2161.111, \"y\": 785.519},\n",
+ " {\"x\": 2002.115, \"y\": 894.647},\n",
+ " {\"x\": 1838.456, \"y\": 877.874},\n",
+ " {\"x\": 1436.53, \"y\": 874.636},\n",
+ " {\"x\": 1411.403, \"y\": 758.579},\n",
+ " {\"x\": 1353.853, \"y\": 751.74},\n",
+ " {\"x\": 1345.264, \"y\": 453.461},\n",
+ " {\"x\": 1426.011, \"y\": 421.129},\n",
+ " {\"x\": 1489.581, \"y\": 183.934}\n",
+ " ]\n",
"}"
],
"cell_type": "code",
@@ -238,22 +358,26 @@
{
"metadata": {},
"source": [
- "## Create Predictions\n",
- "* Loop over data_rows, make predictions, and create ndjson"
+ "### Classification: Free-form text"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "# --- setup dataset ---\n",
- "# load mapillary sample\n",
- "sample_csv_url = \"https://raw.githubusercontent.com/Labelbox/labelbox-python/master/examples/assets/mapillary_sample.csv\"\n",
- "with requests.get(sample_csv_url, stream=True) as r:\n",
- " image_data = [\n",
- " row.split(',')\n",
- " for row in (line.decode('utf-8') for line in r.iter_lines())\n",
- " ]"
+ "text_annotation_ndjson = {\n",
+ " \"name\": \"free_text\",\n",
+ " \"answer\": \"sample text\",\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"confidence\": 0.5\n",
+ "}"
],
"cell_type": "code",
"outputs": [],
@@ -262,42 +386,27 @@
{
"metadata": {},
"source": [
- "predictions = list()\n",
- "for (image_url, external_id) in notebook.tqdm(image_data):\n",
- " image = lb_types.ImageData(url=image_url, external_id=external_id)\n",
- " height, width = image.value.shape[:2]\n",
- " prediction = predict(np.array([image.im_bytes]),\n",
- " min_score=0.5,\n",
- " height=height,\n",
- " width=width)\n",
- " boxes, classes, seg_masks = prediction[\"boxes\"], prediction[\n",
- " \"class_indices\"], prediction[\"seg_masks\"]\n",
- " annotations = []\n",
- " for box, class_idx, seg in zip(boxes, classes, seg_masks):\n",
- " if class_idx in class_mappings:\n",
- " class_info = class_mappings.get(class_idx)\n",
- " if class_info['kind'] == lb.Tool.Type.POLYGON:\n",
- " contours = measure.find_contours(seg, 0.5)\n",
- " pts = contours[0].astype(np.int32)\n",
- " value = lb_types.Polygon(points=[\n",
- " lb_types.Point(x=x, y=y) for x, y in np.roll(pts, 1, axis=-1)\n",
- " ])\n",
- " elif class_info['kind'] == lb.Tool.Type.BBOX:\n",
- " value = lb_types.Rectangle(start=lb_types.Point(x=box[1], y=box[0]),\n",
- " end=lb_types.Point(x=box[3], y=box[2]))\n",
- " elif class_info['kind'] == lb.Tool.Type.POINT:\n",
- " value = lb_types.Point(x=(box[1] + box[3]) / 2.,\n",
- " y=(box[0] + box[2]) / 2.)\n",
- " elif class_info['kind'] == lb.Tool.Type.SEGMENTATION:\n",
- " value = lb_types.Mask(mask=lb_types.MaskData.from_2D_arr(seg *\n",
- " class_info['color']),\n",
- " color=(class_info['color'],) * 3)\n",
- " else:\n",
- " raise ValueError(\n",
- " f\"Unsupported kind found. {class_info['kind']}\")\n",
- " annotations.append(\n",
- " lb_types.ObjectAnnotation(name=class_info['name'], value=value))\n",
- " predictions.append(lb_types.Label(data=image, annotations=annotations))"
+ "### Point"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "point_prediction_ndjson = {\n",
+ " \"name\": \"point\",\n",
+ " \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"classifications\": [],\n",
+ " \"point\": {\"x\": 1166.606, \"y\": 1441.768}\n",
+ "}"
],
"cell_type": "code",
"outputs": [],
@@ -306,17 +415,49 @@
{
"metadata": {},
"source": [
- "## Setup a project"
+ "### Polyline"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "tools = []\n",
- "for target in class_mappings.values():\n",
- " tools.append(lb.Tool(tool=target['kind'], name=target[\"name\"]))\n",
- "ontology_builder = lb.OntologyBuilder(tools=tools)"
+ "polyline_prediction_ndjson = {\n",
+ " \"name\": \"polyline\",\n",
+ " \"confidence\":0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " \"classifications\": [],\n",
+ " \"line\": [\n",
+ " {\"x\": 2534.353, \"y\": 249.471},\n",
+ " {\"x\": 2429.492, \"y\": 182.092},\n",
+ " {\"x\": 2294.322, \"y\": 221.962},\n",
+ " {\"x\": 2224.491, \"y\": 180.463},\n",
+ " {\"x\": 2136.123, \"y\": 204.716},\n",
+ " {\"x\": 1712.247, \"y\": 173.949},\n",
+ " {\"x\": 1703.838, \"y\": 84.438},\n",
+ " {\"x\": 1579.772, \"y\": 82.61},\n",
+ " {\"x\": 1583.442, \"y\": 167.552},\n",
+ " {\"x\": 1478.869, \"y\": 164.903},\n",
+ " {\"x\": 1418.941, \"y\": 318.149},\n",
+ " {\"x\": 1243.128, \"y\": 400.815},\n",
+ " {\"x\": 1022.067, \"y\": 319.007},\n",
+ " {\"x\": 892.367, \"y\": 379.216},\n",
+ " {\"x\": 670.273, \"y\": 364.408},\n",
+ " {\"x\": 613.114, \"y\": 288.16},\n",
+ " {\"x\": 377.559, \"y\": 238.251},\n",
+ " {\"x\": 368.087, \"y\": 185.064},\n",
+ " {\"x\": 246.557, \"y\": 167.286},\n",
+ " {\"x\": 236.648, \"y\": 285.61},\n",
+ " {\"x\": 90.929, \"y\": 326.412}\n",
+ " ]\n",
+ "}\n"
],
"cell_type": "code",
"outputs": [],
@@ -325,16 +466,32 @@
{
"metadata": {},
"source": [
- "print(f\"Setting up: {PROJECT_NAME}\")\n",
+ "## Step 1: Import data rows into Catalog"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "# send a sample image as batch to the project\n",
+ "global_key = \"2560px-Kitano_Street_Kobe01s5s4110.jpeg\"\n",
+ "test_img_urls = {\n",
+ " \"row_data\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg\",\n",
+ " \"global_key\": global_key\n",
+ "}\n",
+ "\n",
+ "dataset = client.create_dataset(name=\"Custom metrics demo\", iam_integration=None)\n",
+ "task = dataset.create_data_rows([test_img_urls])\n",
"\n",
- "project = client.create_project(name=PROJECT_NAME, media_type=lb.MediaType.Image)\n",
- "editor = next(\n",
- " client.get_labeling_frontends(where=lb.LabelingFrontend.name == \"Editor\"))\n",
- "project.setup(editor, ontology_builder.asdict())\n",
+ "print(f\"Failed data rows: {task.failed_data_rows}\")\n",
+ "print(f\"Errors: {task.errors}\")\n",
"\n",
- "dataset = client.create_dataset(name=\"Mapillary Diagnostics Demo\")\n",
- "print(f\"Dataset Created: {dataset.uid}\")\n",
- "project.create_batches_from_dataset(\"batch\", dataset.uid)"
+ "if task.errors:\n",
+ " for error in task.errors:\n",
+ " if 'Duplicate global key' in error['message'] and dataset.row_count == 0:\n",
+ " # If the global key already exists in the workspace the dataset will be created empty, so we can delete it.\n",
+ " print(f\"Deleting empty dataset: {dataset}\")\n",
+ " dataset.delete()"
],
"cell_type": "code",
"outputs": [],
@@ -343,21 +500,99 @@
{
"metadata": {},
"source": [
- "## Prepare for upload\n",
- "* Our local annotations need the following:\n",
- " 1. signed url for segmentation masks\n",
- " 2. data rows in labelbox\n",
- " 3. feature schema ids"
+ "## Step 2: Create/select an Ontology for your model predictions\n",
+ "Your project should have the correct ontology setup with all the tools and classifications supported for your annotations, and the tool names and classification instructions should match the name/instructions fields in your annotations to ensure the correct feature schemas are matched.\n"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "signer = lambda _bytes: client.upload_data(content=_bytes, sign=True)\n",
- "predictions.add_url_to_masks(signer) \\\n",
- " .add_url_to_data(signer) \\\n",
- " .add_to_dataset(dataset, client.upload_data)"
+ "ontology_builder = lb.OntologyBuilder(\n",
+ " classifications=[ # List of Classification objects\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.RADIO,\n",
+ " name=\"radio_question\",\n",
+ " options=[\n",
+ " lb.Option(value=\"first_radio_answer\"),\n",
+ " lb.Option(value=\"second_radio_answer\")\n",
+ " ]\n",
+ " ),\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.CHECKLIST,\n",
+ " name=\"checklist_question\",\n",
+ " options=[\n",
+ " lb.Option(value=\"first_checklist_answer\"),\n",
+ " lb.Option(value=\"second_checklist_answer\")\n",
+ " ]\n",
+ " ),\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.TEXT,\n",
+ " name=\"free_text\"\n",
+ " ),\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.RADIO,\n",
+ " name=\"nested_radio_question\",\n",
+ " options=[\n",
+ " lb.Option(\"first_radio_answer\",\n",
+ " options=[\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.RADIO,\n",
+ " name=\"sub_radio_question\",\n",
+ " options=[lb.Option(\"first_sub_radio_answer\")]\n",
+ " )\n",
+ " ]\n",
+ " )\n",
+ " ]\n",
+ " ),\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.CHECKLIST,\n",
+ " name=\"nested_checklist_question\",\n",
+ " options=[\n",
+ " lb.Option(\"first_checklist_answer\",\n",
+ " options=[\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.CHECKLIST,\n",
+ " name=\"sub_checklist_question\",\n",
+ " options=[lb.Option(\"first_sub_checklist_answer\")]\n",
+ " )\n",
+ " ]\n",
+ " )\n",
+ " ]\n",
+ " ),\n",
+ " ],\n",
+ " tools=[ # List of tools\n",
+ " lb.Tool(\n",
+ " tool=lb.Tool.Type.BBOX,\n",
+ " name=\"bounding_box\"),\n",
+ " lb.Tool(\n",
+ " tool=lb.Tool.Type.BBOX,\n",
+ " name=\"bbox_with_radio_subclass\",\n",
+ " classifications=[\n",
+ " lb.Classification(\n",
+ " class_type=lb.Classification.Type.RADIO,\n",
+ " name=\"sub_radio_question\",\n",
+ " options=[\n",
+ " lb.Option(value=\"first_sub_radio_answer\")\n",
+ " ]\n",
+ " ),\n",
+ " ]\n",
+ " ),\n",
+ " lb.Tool(\n",
+ " tool=lb.Tool.Type.POLYGON,\n",
+ " name=\"polygon\"),\n",
+ " \t lb.Tool(\n",
+ " tool=lb.Tool.Type.POINT,\n",
+ " name=\"point\"),\n",
+ " lb.Tool(\n",
+ " tool=lb.Tool.Type.LINE,\n",
+ " name=\"polyline\")]\n",
+ ")\n",
+ "\n",
+ "ontology = client.create_ontology(\"Image Prediction Import Demo\",\n",
+ " ontology_builder.asdict(),\n",
+ " media_type=lb.MediaType.Image\n",
+ " )"
],
"cell_type": "code",
"outputs": [],
@@ -366,24 +601,18 @@
{
"metadata": {},
"source": [
- "## **Optional** - Create labels with [Model Assisted Labeling](https://docs.labelbox.com/en/core-concepts/model-assisted-labeling)\n",
- "\n",
- "* Pre-label image so that we can quickly create ground truth\n",
- "* Create ground truth data for Model Diagnostics\n",
- "* Click on link below to label"
+ "## Step 3: Create a Model and Model Run"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "RUN_MAL = True\n",
- "if RUN_MAL:\n",
- " project.enable_model_assisted_labeling()\n",
- " upload_task = lb.MALPredictionImport.create_from_objects(\n",
- " client, project.uid, f'mal-import-{uuid.uuid4()}', predictions)\n",
- " upload_task.wait_until_done()\n",
- " print(upload_task.state, '\\n')"
+ "# create Model\n",
+ "model = client.create_model(name=\"model_with_aggregated_custom_metrics\" + str(uuid.uuid4()),\n",
+ " ontology_id=ontology.uid)\n",
+ "# create Model Run\n",
+ "model_run = model.create_model_run(\"iteration 1\")"
],
"cell_type": "code",
"outputs": [],
@@ -392,7 +621,14 @@
{
"metadata": {},
"source": [
- "print(f\"https://app.labelbox.com/go-label/{project.uid}\")"
+ "## Step 4: Send data rows to the Model Run"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "model_run.upsert_data_rows(global_keys=[global_key])"
],
"cell_type": "code",
"outputs": [],
@@ -401,19 +637,42 @@
{
"metadata": {},
"source": [
- "## Export Labels\n",
+ "## Step 5. Create the predictions payload\n",
+ "\n",
+ "Create the prediction payload using the snippets of code in ***Supported Predictions*** section.\n",
"\n",
- "We do not support `Skipped` labels and have a limit of **2000**"
+ "The resulting label_ndjson should have exactly the same content for predictions that are supported by both (with exception of the uuid strings that are generated)"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "MAX_LABELS = 2000\n",
- "labels = [\n",
- " l for idx, l in enumerate(project.label_generator()) if idx < MAX_LABELS\n",
- "]"
+ "If using NDJSON"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "label_prediction_ndjson = []\n",
+ "\n",
+ "for annot in [\n",
+ " radio_prediction_ndjson,\n",
+ " checklist_prediction_ndjson,\n",
+ " bbox_prediction_ndjson,\n",
+ " bbox_with_radio_subclass_prediction_ndjson,\n",
+ " polygon_prediction_ndjson,\n",
+ " point_prediction_ndjson,\n",
+ " polyline_prediction_ndjson,\n",
+ " text_annotation_ndjson,\n",
+ " nested_radio_prediction_ndjson,\n",
+ " nested_checklist_prediction_ndjson\n",
+ "]:\n",
+ " annot.update({\n",
+ " \"dataRow\": {\"globalKey\": global_key}\n",
+ " })\n",
+ " label_prediction_ndjson.append(annot)"
],
"cell_type": "code",
"outputs": [],
@@ -422,16 +681,20 @@
{
"metadata": {},
"source": [
- "## Setup Model & Model Run"
+ "## Step 6. Upload the predictions payload to the Model Run"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "lb_model = client.create_model(name=MODEL_NAME,\n",
- " ontology_id=project.ontology().uid)\n",
- "lb_model_run = lb_model.create_model_run(MODEL_VERSION)"
+ "# Upload the prediction label to the Model Run\n",
+ "upload_job_prediction = model_run.add_predictions(\n",
+ " name=\"prediction_upload_job\"+str(uuid.uuid4()),\n",
+ " predictions=label_prediction_ndjson)\n",
+ "\n",
+ "# Errors will appear for prediction uploads that failed.\n",
+ "print(\"Errors:\", upload_job_prediction.errors)"
],
"cell_type": "code",
"outputs": [],
@@ -440,14 +703,26 @@
{
"metadata": {},
"source": [
- "Select label ids to upload"
+ "## Step 7: Send annotations to a model run\n",
+ "To visualize both annotations and predictions in the model run we will create a project with ground truth annotations.\n",
+ "To send annotations to a Model Run, we must first import them into a project, create a label payload and then send them to the Model Run."
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "lb_model_run.upsert_labels([label.uid for label in labels])"
+ "##### 7.1. Create a labelbox project"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "# Create a Labelbox project\n",
+ "project = client.create_project(name=\"image_prediction_many_kinds\",\n",
+ " media_type=lb.MediaType.Image)\n",
+ "project.setup_editor(ontology)"
],
"cell_type": "code",
"outputs": [],
@@ -456,58 +731,137 @@
{
"metadata": {},
"source": [
- "### Compute Metrics\n",
- "* First get pairs of labels and predictions"
+ "##### 7.2. Create a batch to send to the project"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "* Create helper functions for our metric\n",
- "* All functions will accept ground truth and prediction annotations"
+ "project.create_batch(\n",
+ " \"batch_predictions_demo\", # Each batch in a project must have a unique name\n",
+ " global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys\n",
+ " priority=5 # priority between 1(Highest) - 5(lowest)\n",
+ ")"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
+ },
+ {
+ "metadata": {},
+ "source": [
+ "##### 7.3 Create the annotations payload"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "from shapely.ops import cascaded_union\n",
+ "########### Annotations ###########\n",
+ "radio_annotation = lb_types.ClassificationAnnotation(\n",
+ " name=\"radio_question\",\n",
+ " value=lb_types.Radio(answer=lb_types.ClassificationAnswer(\n",
+ " name=\"second_radio_answer\"))\n",
+ ")\n",
"\n",
+ "nested_radio_annotation = lb_types.ClassificationAnnotation(\n",
+ " name=\"nested_radio_question\",\n",
+ " value=lb_types.Radio(\n",
+ " answer=lb_types.ClassificationAnswer(\n",
+ " name=\"first_radio_answer\",\n",
+ " classifications=[\n",
+ " lb_types.ClassificationAnnotation(\n",
+ " name=\"sub_radio_question\",\n",
+ " value=lb_types.Radio(\n",
+ " answer=lb_types.ClassificationAnswer(\n",
+ " name=\"first_sub_radio_answer\"\n",
+ " )\n",
+ " )\n",
+ " )\n",
+ " ]\n",
+ " )\n",
+ " )\n",
+ ")\n",
"\n",
- "def nearby_cars_iou(ground_truths, predictions, area_threshold=17000):\n",
- " \"\"\"\n",
- " Metric to track the iou score for cars that are nearby (determined by pixel size).\n",
- " \n",
- " This might be useful to investigate why the model poorly when vehicles are nearby.\n",
- " We might care a lot about optimizing this metric because our self driving car needs to \n",
- " be aware of its immediate surroundings for safety reasons.\n",
- " \"\"\"\n",
- " ground_truths = [gt for gt in ground_truths if gt.name == 'car']\n",
- " predictions = [pred for pred in predictions if pred.name == 'car']\n",
- " ground_truths = cascaded_union([\n",
- " gt.value.shapely\n",
- " for gt in ground_truths\n",
- " if gt.value.shapely.area > area_threshold\n",
- " ])\n",
- " predictions = cascaded_union([\n",
- " pred.value.shapely\n",
- " for pred in predictions\n",
- " if pred.value.shapely.area > area_threshold\n",
+ "nested_checklist_annotation = lb_types.ClassificationAnnotation(\n",
+ " name=\"nested_checklist_question\",\n",
+ " value=lb_types.Checklist(\n",
+ " answer=[lb_types.ClassificationAnswer(\n",
+ " name=\"first_checklist_answer\",\n",
+ " classifications=[\n",
+ " lb_types.ClassificationAnnotation(\n",
+ " name=\"sub_checklist_question\",\n",
+ " value=lb_types.Checklist(\n",
+ " answer=[lb_types.ClassificationAnswer(\n",
+ " name=\"first_sub_checklist_answer\"\n",
+ " )]\n",
+ " ))\n",
+ " ]\n",
+ " )]\n",
+ " )\n",
+ ")\n",
+ "\n",
+ "checklist_annotation = lb_types.ClassificationAnnotation(\n",
+ " name=\"checklist_question\",\n",
+ " value=lb_types.Checklist(answer=[\n",
+ " lb_types.ClassificationAnswer(name=\"first_checklist_answer\"),\n",
+ " lb_types.ClassificationAnswer(name=\"second_checklist_answer\")\n",
" ])\n",
- " union = ground_truths.union(predictions).area\n",
- " # If there is no prediction or label then the score is undefined\n",
- " if union == 0:\n",
- " return []\n",
- " return [\n",
- " ScalarMetric(\n",
- " value=ground_truths.intersection(predictions).area / union,\n",
- " metric_name=\"iou\",\n",
- " feature_name=\"car\",\n",
- " subclass_name=\n",
- " \"nearby\" # Doesn't necessarily need to be a subclass in the ontology\n",
- " )\n",
- " ]"
+ ")\n",
+ "\n",
+ "bbox_annotation = lb_types.ObjectAnnotation(\n",
+ " name = \"bounding_box\",\n",
+ " value=lb_types.Rectangle(\n",
+ " start=lb_types.Point(x=1690, y=977), # x = left, y = top\n",
+ " end=lb_types.Point(x=1915, y=1307), # x= left + width , y = top + height\n",
+ " )\n",
+ ")\n",
+ "\n",
+ "bbox_with_radio_subclass_annotation = lb_types.ObjectAnnotation(\n",
+ " name=\"bbox_with_radio_subclass\",\n",
+ " value=lb_types.Rectangle(\n",
+ " start=lb_types.Point(x=541, y=933), # x = left, y = top\n",
+ " end=lb_types.Point(x=871, y=1124), # x= left + width , y = top + height\n",
+ " ),\n",
+ " classifications=[\n",
+ " \tlb_types.ClassificationAnnotation(\n",
+ " \tname=\"sub_radio_question\",\n",
+ " \t\tvalue=lb_types.Radio(answer=lb_types.ClassificationAnswer(name=\"first_sub_radio_answer\", confidence=0.5))\n",
+ " )\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "polygon_annotation = lb_types.ObjectAnnotation(\n",
+ " name = \"polygon\",\n",
+ " value=lb_types.Polygon(\n",
+ " points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419),\n",
+ " lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519),\n",
+ " lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579),\n",
+ " lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)]\n",
+ " ),\n",
+ ")\n",
+ "\n",
+ "text_annotation = lb_types.ClassificationAnnotation(\n",
+ " name=\"free_text\",\n",
+ " value=lb_types.Text(answer=\"sample text\")\n",
+ ")\n",
+ "\n",
+ "point_annotation = lb_types.ObjectAnnotation(\n",
+ " name = \"point\",\n",
+ " value = lb_types.Point(x=1166.606, y=1441.768),\n",
+ ")\n",
+ "\n",
+ "polyline_annotation = lb_types.ObjectAnnotation(\n",
+ " name = \"polyline\",\n",
+ " value=lb_types.Line(\n",
+ " points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716),\n",
+ " lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552),\n",
+ " lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007),\n",
+ " lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251),\n",
+ " lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)]\n",
+ " )\n",
+ ")"
],
"cell_type": "code",
"outputs": [],
@@ -516,26 +870,56 @@
{
"metadata": {},
"source": [
- "* Compute and sssign each metric to prediction label"
+ "##### 7.4. Create the label object"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "# Metric functions expect to be provided labels and predictions that correspond to the same data image/video frame\n",
- "# This get_label_pairs function uses the datarow id to achieve this.\n",
- "pairs = get_label_pairs(labels, predictions, filter_mismatch=True)\n",
- "for (ground_truth, prediction) in pairs.values():\n",
- " metrics = []\n",
- " metrics.extend(\n",
- " feature_miou_metric(ground_truth.annotations, prediction.annotations))\n",
- " metrics.extend(\n",
- " feature_confusion_matrix_metric(ground_truth.annotations,\n",
- " prediction.annotations))\n",
- " metrics.extend(\n",
- " nearby_cars_iou(ground_truth.annotations, prediction.annotations))\n",
- " prediction.annotations.extend(metrics)"
+ "# Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations\n",
+ "label = []\n",
+ "annotations = [\n",
+ " radio_annotation,\n",
+ " nested_radio_annotation,\n",
+ " checklist_annotation,\n",
+ " nested_checklist_annotation,\n",
+ " text_annotation,\n",
+ " bbox_annotation,\n",
+ " bbox_with_radio_subclass_annotation,\n",
+ " polygon_annotation,\n",
+ " point_annotation,\n",
+ " polyline_annotation\n",
+ "]\n",
+ "label.append(\n",
+ " lb_types.Label(\n",
+ " data=lb_types.ImageData(global_key=global_key),\n",
+ " annotations=annotations))"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
+ },
+ {
+ "metadata": {},
+ "source": [
+ "##### 7.5. Upload annotations to the project using Label Import"
+ ],
+ "cell_type": "markdown"
+ },
+ {
+ "metadata": {},
+ "source": [
+ "upload_job_annotation = lb.LabelImport.create_from_objects(\n",
+ " client = client,\n",
+ " project_id = project.uid,\n",
+ " name=\"annotation_import_\" + str(uuid.uuid4()),\n",
+ " labels=label)\n",
+ "\n",
+ "upload_job_annotation.wait_until_done()\n",
+ "# Errors will appear for annotation uploads that failed.\n",
+ "print(\"Errors:\", upload_job_annotation.errors)\n",
+ "print(\"Status of uploads: \", upload_job_annotation.statuses)"
],
"cell_type": "code",
"outputs": [],
@@ -544,17 +928,14 @@
{
"metadata": {},
"source": [
- "### Upload to Labelbox"
+ "##### 7.6 Send the annotations to the Model Run"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "upload_task = lb_model_run.add_predictions(\n",
- " f'diagnostics-import-{uuid.uuid4()}', predictions)\n",
- "upload_task.wait_until_done()\n",
- "print(upload_task.state)"
+ "model_run.upsert_labels(project_id=project.uid)"
],
"cell_type": "code",
"outputs": [],
@@ -563,17 +944,15 @@
{
"metadata": {},
"source": [
- "### Open Model Run"
+ "## Optional deletions for cleanup\n"
],
"cell_type": "markdown"
},
{
"metadata": {},
"source": [
- "for idx, model_run_data_row in enumerate(lb_model_run.model_run_data_rows()):\n",
- " if idx == 5:\n",
- " break\n",
- " print(model_run_data_row.url)"
+ "# project.delete()\n",
+ "# dataset.delete()"
],
"cell_type": "code",
"outputs": [],
diff --git a/examples/project_configuration/webhooks.ipynb b/examples/project_configuration/webhooks.ipynb
index 904e13f91..ec54de860 100644
--- a/examples/project_configuration/webhooks.ipynb
+++ b/examples/project_configuration/webhooks.ipynb
@@ -1,16 +1,18 @@
{
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ "metadata": {},
"cells": [
{
- "cell_type": "markdown",
"metadata": {},
"source": [
"\n",
" \n",
" | "
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "markdown",
"metadata": {},
"source": [
"\n",
@@ -22,32 +24,28 @@
"\n",
" | "
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "markdown",
"metadata": {},
"source": [
"# Webhook Configuration"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "markdown",
- "id": "18fc16be",
"metadata": {},
"source": [
"Webhooks are supported for the following events:\n",
"* label_created\n",
"* label_updated\n",
"* label_deleted"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "ad5486cd",
"metadata": {},
- "outputs": [],
"source": [
"!pip install labelbox\n",
"!pip install requests\n",
@@ -55,14 +53,13 @@
"!pip install hashlib\n",
"!pip install flask\n",
"!pip install Werkzeug"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "a2a8f3b2",
"metadata": {},
- "outputs": [],
"source": [
"import labelbox as lb\n",
"from flask import Flask, request\n",
@@ -75,14 +72,13 @@
"import os\n",
"from getpass import getpass\n",
"import socket"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "99a23b2b",
"metadata": {},
- "outputs": [],
"source": [
"# If you don't want to give google access to drive you can skip this cell\n",
"# and manually set `API_KEY` below.\n",
@@ -99,46 +95,44 @@
" API_KEY = getpass(\"Please enter your labelbox api key\")\n",
" if COLAB:\n",
" envvar_handler.add_env(\"LABELBOX_API_KEY\", API_KEY)"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "ee93a4ab",
"metadata": {},
- "outputs": [],
"source": [
"# Set this to a project that you want to use for the webhook\n",
"PROJECT_ID = \"\"\n",
"# Only update this if you have an on-prem deployment\n",
"ENDPOINT = \"https://api.labelbox.com/graphql\""
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "a161946b",
"metadata": {},
- "outputs": [],
"source": [
"client = lb.Client(api_key=API_KEY, endpoint=ENDPOINT)"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "29890bf8",
"metadata": {},
- "outputs": [],
"source": [
"# We are using port 3001 for this example.\n",
"# Feel free to set to whatever port you want\n",
"WH_PORT = 3001"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "markdown",
- "id": "8e48bb5f",
"metadata": {},
"source": [
"### Configure NGROK (Optional)\n",
@@ -149,44 +143,38 @@
"2. Download ngrok and extract the zip file\n",
"3. Add ngrok to your path\n",
"4. Add the authtoken `ngrok authtoken `"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "005702a2",
"metadata": {},
- "outputs": [],
"source": [
"if not COLAB:\n",
" os.system(f\"ngrok http {WH_PORT} &\")"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "markdown",
- "id": "4417e81b",
"metadata": {},
"source": [
"### Configure server to receive requests"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "4bbfd57c",
"metadata": {},
- "outputs": [],
"source": [
"# This can be any secret that matches your webhook config (we will set later)\n",
"secret = b\"example_secret\""
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "5b98f952",
"metadata": {},
- "outputs": [],
"source": [
"app = Flask(__name__)\n",
"\n",
@@ -217,49 +205,44 @@
"\n",
"thread = threading.Thread(target=lambda: run_simple(\"0.0.0.0\", WH_PORT, app))\n",
"thread.start()"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "markdown",
- "id": "2a6fce6d",
"metadata": {},
"source": [
"#### Test server"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "c53f862c",
"metadata": {},
- "outputs": [],
"source": [
"print(requests.get(\"http://localhost:3001\").text)"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "markdown",
- "id": "b84254a8",
"metadata": {},
"source": [
"### Create Webhook"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "markdown",
- "id": "916f3436",
"metadata": {},
"source": [
"- Set ip address if your ip is publicly accessible.\n",
"- Otherwise use the following to get ngrok public_url"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "41832282",
"metadata": {},
- "outputs": [],
"source": [
"if not COLAB:\n",
" res = requests.get(\"http://localhost:4040/api/tunnels\")\n",
@@ -274,14 +257,13 @@
"else:\n",
" public_url = f\"http://{socket.gethostbyname(socket.getfqdn(socket.gethostname()))}\"\n",
"print(public_url)"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "2f18123e",
"metadata": {},
- "outputs": [],
"source": [
"# Set project to limit the scope to a single project\n",
"project = client.get_project(PROJECT_ID)\n",
@@ -292,34 +274,31 @@
" url=public_url,\n",
" secret=secret.decode(),\n",
" project=project)"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "b9226f8f",
"metadata": {},
- "outputs": [],
"source": [
"# Ok so we should be configured assuming everything is setup correctly.\n",
"# Go to the following url and make a new label to see if it works\n",
"print(f\"https://app.labelbox.com/projects/{PROJECT_ID}\")"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "markdown",
- "id": "bae73fb4",
"metadata": {},
"source": [
"### Update Webhook"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "d7070ab0",
"metadata": {},
- "outputs": [],
"source": [
"# url, topics, and status can all be updated\n",
"updated_url = f\"{public_url}/webhook-endpoint\"\n",
@@ -328,20 +307,20 @@
"# Go to the following url and try one last time.\n",
"# Any supported action should work (create, delete, or update a label)\n",
"print(f\"https://app.labelbox.com/projects/{PROJECT_ID}\")"
- ]
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
},
{
- "cell_type": "markdown",
"metadata": {},
"source": [
"### List and delete all webhooks"
- ]
+ ],
+ "cell_type": "markdown"
},
{
- "cell_type": "code",
- "execution_count": null,
"metadata": {},
- "outputs": [],
"source": [
"# DELETE:\n",
"webhook.update(status=lb.Webhook.Status.INACTIVE.value)\n",
@@ -356,14 +335,10 @@
"# for webhook in webhooks:\n",
"# print(webhook)\n",
"# webhook.update(status = lb.Webhook.Status.INACTIVE.value)"
- ]
- }
- ],
- "metadata": {
- "language_info": {
- "name": "python"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
}
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}
+ ]
+}
\ No newline at end of file