diff --git a/examples/model_diagnostics/custom_metrics_basics.ipynb b/examples/model_experiments/model_diagnostics/custom_metrics_basics.ipynb
similarity index 97%
rename from examples/model_diagnostics/custom_metrics_basics.ipynb
rename to examples/model_experiments/model_diagnostics/custom_metrics_basics.ipynb
index 6c5c54ffb..ea7312f6a 100644
--- a/examples/model_diagnostics/custom_metrics_basics.ipynb
+++ b/examples/model_experiments/model_diagnostics/custom_metrics_basics.ipynb
@@ -16,27 +16,17 @@
"metadata": {},
"source": [
"| \n",
- "\n",
" | \n",
"\n",
"\n",
- "\n",
" | "
],
"cell_type": "markdown"
},
- {
- "metadata": {},
- "source": [
- " \n",
- "\n",
- " ![](\"https://labelbox.com/blog/content/images/2021/02/logo-v4.svg\") \n",
- " | "
- ],
- "cell_type": "markdown"
- },
{
"metadata": {},
"source": [
diff --git a/examples/model_diagnostics/custom_metrics_demo.ipynb b/examples/model_experiments/model_diagnostics/custom_metrics_demo.ipynb
similarity index 70%
rename from examples/model_diagnostics/custom_metrics_demo.ipynb
rename to examples/model_experiments/model_diagnostics/custom_metrics_demo.ipynb
index 42b8575d1..d18119c2c 100644
--- a/examples/model_diagnostics/custom_metrics_demo.ipynb
+++ b/examples/model_experiments/model_diagnostics/custom_metrics_demo.ipynb
@@ -17,12 +17,12 @@
"source": [
"\n",
"\n",
- "\n",
" | \n",
"\n",
"\n",
- "\n",
" | "
],
@@ -105,6 +105,20 @@
{
"metadata": {},
"source": [
+ "radio_prediction = lb_types.ClassificationAnnotation(\n",
+ " name=\"radio_question\",\n",
+ " value=lb_types.Radio(\n",
+ " answer = lb_types.ClassificationAnswer(\n",
+ " name = \"second_radio_answer\", confidence=0.5,\n",
+ " custom_metrics=[{ 'name': 'iou', 'value': 0.1 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }],\n",
+ " )\n",
+ " )\n",
+ ")\n",
"\n",
"# NDJSON\n",
"radio_prediction_ndjson = {\n",
@@ -135,10 +149,42 @@
{
"metadata": {},
"source": [
+ "checklist_prediction = lb_types.ClassificationAnnotation(\n",
+ " name=\"checklist_question\",\n",
+ " value=lb_types.Checklist(\n",
+ " answer = [\n",
+ " lb_types.ClassificationAnswer(\n",
+ " name = \"first_checklist_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics=[\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " ),\n",
+ " lb_types.ClassificationAnswer(\n",
+ " name = \"second_checklist_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics= [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " )\n",
+ " ]\n",
+ " )\n",
+ " )\n",
"checklist_prediction_ndjson = {\n",
" 'name': 'checklist_question',\n",
" 'answer': [\n",
- " {'name': 'first_checklist_answer' , 'confidence': 0.5, 'customMetrics': [\n",
+ " {'name': 'first_checklist_answer' , 'confidence': 0.5,\n",
+ " 'customMetrics': [\n",
" { 'name': 'iou', 'value': 0.5 },\n",
" { 'name': 'f1', 'value': 0.33 },\n",
" { 'name': 'precision', 'value': 0.55 },\n",
@@ -146,7 +192,8 @@
" { 'name': 'tagsCount', 'value': 43 },\n",
" { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
" ]},\n",
- " {'name': 'second_checklist_answer', 'confidence': 0.5, 'customMetrics': [\n",
+ " {'name': 'second_checklist_answer', 'confidence': 0.5,\n",
+ " 'customMetrics': [\n",
" { 'name': 'iou', 'value': 0.5 },\n",
" { 'name': 'f1', 'value': 0.33 },\n",
" { 'name': 'precision', 'value': 0.55 },\n",
@@ -171,6 +218,43 @@
{
"metadata": {},
"source": [
+ "nested_radio_prediction = lb_types.ClassificationAnnotation(\n",
+ " name=\"nested_radio_question\",\n",
+ " value=lb_types.Radio(\n",
+ " answer=lb_types.ClassificationAnswer(\n",
+ " name=\"first_radio_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics=[\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " classifications=[\n",
+ " lb_types.ClassificationAnnotation(\n",
+ " name=\"sub_radio_question\",\n",
+ " value=lb_types.Radio(\n",
+ " answer=lb_types.ClassificationAnswer(\n",
+ " name=\"first_sub_radio_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics=[\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " )\n",
+ " )\n",
+ " )\n",
+ " ]\n",
+ " )\n",
+ " )\n",
+ ")\n",
+ "\n",
"nested_radio_prediction_ndjson = {\n",
" \"name\": \"nested_radio_question\",\n",
" \"confidence\": 0.5 ,\n",
@@ -201,6 +285,42 @@
" ]\n",
"}\n",
"\n",
+ "nested_checklist_prediction = lb_types.ClassificationAnnotation(\n",
+ " name=\"nested_checklist_question\",\n",
+ " value=lb_types.Checklist(\n",
+ " answer=[lb_types.ClassificationAnswer(\n",
+ " name=\"first_checklist_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics = [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " classifications=[\n",
+ " lb_types.ClassificationAnnotation(\n",
+ " name=\"sub_checklist_question\",\n",
+ " value=lb_types.Checklist(\n",
+ " answer=[lb_types.ClassificationAnswer(\n",
+ " name=\"first_sub_checklist_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics= [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " )]\n",
+ " ))\n",
+ " ]\n",
+ " )]\n",
+ " )\n",
+ ")\n",
+ "\n",
"nested_checklist_prediction_ndjson = {\n",
" \"name\": \"nested_checklist_question\",\n",
" \"confidence\": 0.5 ,\n",
@@ -246,6 +366,23 @@
{
"metadata": {},
"source": [
+ "bbox_prediction = lb_types.ObjectAnnotation(\n",
+ " name = \"bounding_box\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics= [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " value=lb_types.Rectangle(\n",
+ " start=lb_types.Point(x=1690, y=977), # x = left, y = top\n",
+ " end=lb_types.Point(x=1915, y=1307), # x= left + width , y = top + height\n",
+ " )\n",
+ ")\n",
+ "\n",
"\n",
"bbox_prediction_ndjson = {\n",
" \"name\": \"bounding_box\",\n",
@@ -280,6 +417,40 @@
{
"metadata": {},
"source": [
+ "bbox_with_radio_subclass_prediction = lb_types.ObjectAnnotation(\n",
+ " name=\"bbox_with_radio_subclass\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics = [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.2 },\n",
+ " { 'name': 'precision', 'value': 0.1 },\n",
+ " { 'name': 'recall', 'value': 0.3 },\n",
+ " { 'name': 'tagsCount', 'value': 23 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " value=lb_types.Rectangle(\n",
+ " start=lb_types.Point(x=541, y=933), # x = left, y = top\n",
+ " end=lb_types.Point(x=871, y=1124), # x= left + width , y = top + height\n",
+ " ),\n",
+ " classifications=[\n",
+ " \tlb_types.ClassificationAnnotation(\n",
+ " \tname=\"sub_radio_question\",\n",
+ " \t\tvalue=lb_types.Radio(\n",
+ " answer=lb_types.ClassificationAnswer(\n",
+ " name=\"first_sub_radio_answer\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics =[\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.2 },\n",
+ " { 'name': 'precision', 'value': 0.1 },\n",
+ " { 'name': 'recall', 'value': 0.3 },\n",
+ " { 'name': 'tagsCount', 'value': 23 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " ))\n",
+ " )\n",
+ " ]\n",
+ ")\n",
"## NDJSON\n",
"bbox_with_radio_subclass_prediction_ndjson = {\n",
" \"name\": \"bbox_with_radio_subclass\",\n",
@@ -295,7 +466,16 @@
" \"classifications\": [{\n",
" \"name\": \"sub_radio_question\",\n",
" \"answer\":\n",
- " { \"name\":\"first_sub_radio_answer\", \"confidence\": 0.5}\n",
+ " { \"name\":\"first_sub_radio_answer\",\n",
+ " \"confidence\": 0.5,\n",
+ " 'customMetrics': [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.2 },\n",
+ " { 'name': 'precision', 'value': 0.1 },\n",
+ " { 'name': 'recall', 'value': 0.3 },\n",
+ " { 'name': 'tagsCount', 'value': 23 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]}\n",
"\n",
" }],\n",
" \"bbox\": {\n",
@@ -320,6 +500,26 @@
{
"metadata": {},
"source": [
+ "# Python Anotation\n",
+ "polygon_prediction = lb_types.ObjectAnnotation(\n",
+ " name = \"polygon\",\n",
+ " confidence = 0.5,\n",
+ " custom_metrics = [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " value=lb_types.Polygon(\n",
+ " points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419),\n",
+ " lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519),\n",
+ " lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579),\n",
+ " lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)]\n",
+ " ),\n",
+ ")\n",
+ "\n",
"polygon_prediction_ndjson = {\n",
" \"name\": \"polygon\",\n",
" \"confidence\": 0.5,\n",
@@ -365,6 +565,24 @@
{
"metadata": {},
"source": [
+ "# Python annotation\n",
+ "text_annotation = lb_types.ClassificationAnnotation(\n",
+ " name=\"free_text\",\n",
+ " value=lb_types.Text(\n",
+ " answer=\"sample text\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics = [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ]\n",
+ " )\n",
+ ")\n",
+ "\n",
+ "\n",
"text_annotation_ndjson = {\n",
" \"name\": \"free_text\",\n",
" \"answer\": \"sample text\",\n",
@@ -393,6 +611,21 @@
{
"metadata": {},
"source": [
+ "# Python Annotation\n",
+ "point_prediction = lb_types.ObjectAnnotation(\n",
+ " name = \"point\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics= [\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " value = lb_types.Point(x=1166.606, y=1441.768),\n",
+ ")\n",
+ "\n",
"point_prediction_ndjson = {\n",
" \"name\": \"point\",\n",
" \"confidence\": 0.5,\n",
@@ -422,6 +655,27 @@
{
"metadata": {},
"source": [
+ "polyline_prediction = lb_types.ObjectAnnotation(\n",
+ " name = \"polyline\",\n",
+ " confidence=0.5,\n",
+ " custom_metrics=[\n",
+ " { 'name': 'iou', 'value': 0.5 },\n",
+ " { 'name': 'f1', 'value': 0.33 },\n",
+ " { 'name': 'precision', 'value': 0.55 },\n",
+ " { 'name': 'recall', 'value': 0.33 },\n",
+ " { 'name': 'tagsCount', 'value': 43 },\n",
+ " { 'name': 'metric_with_a_very_long_name', 'value': 0.334332 }\n",
+ " ],\n",
+ " value=lb_types.Line(\n",
+ " points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716),\n",
+ " lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552),\n",
+ " lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007),\n",
+ " lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251),\n",
+ " lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)]\n",
+ " ),\n",
+ ")\n",
+ "\n",
+ "\n",
"polyline_prediction_ndjson = {\n",
" \"name\": \"polyline\",\n",
" \"confidence\":0.5,\n",
@@ -645,6 +899,33 @@
],
"cell_type": "markdown"
},
+ {
+ "metadata": {},
+ "source": [
+ "# Create a Label for predictions\n",
+ "label_prediction = []\n",
+ "label_prediction.append(lb_types.Label(\n",
+ " data=lb_types.ImageData(global_key=global_key),\n",
+ " annotations = [\n",
+ " radio_prediction,\n",
+ " nested_radio_prediction,\n",
+ " checklist_prediction,\n",
+ " nested_checklist_prediction,\n",
+ " bbox_prediction,\n",
+ " bbox_with_radio_subclass_prediction,\n",
+ " polyline_prediction,\n",
+ " polygon_prediction,\n",
+ " point_prediction,\n",
+ " text_annotation\n",
+ "\n",
+ " ]\n",
+ " )\n",
+ ")"
+ ],
+ "cell_type": "code",
+ "outputs": [],
+ "execution_count": null
+ },
{
"metadata": {},
"source": [
@@ -691,7 +972,7 @@
"# Upload the prediction label to the Model Run\n",
"upload_job_prediction = model_run.add_predictions(\n",
" name=\"prediction_upload_job\"+str(uuid.uuid4()),\n",
- " predictions=label_prediction_ndjson)\n",
+ " predictions=label_prediction)\n",
"\n",
"# Errors will appear for prediction uploads that failed.\n",
"print(\"Errors:\", upload_job_prediction.errors)"
diff --git a/examples/model_diagnostics/model_diagnostics_demo.ipynb b/examples/model_experiments/model_diagnostics/model_diagnostics_demo.ipynb
similarity index 98%
rename from examples/model_diagnostics/model_diagnostics_demo.ipynb
rename to examples/model_experiments/model_diagnostics/model_diagnostics_demo.ipynb
index 5c728f979..914ac9801 100644
--- a/examples/model_diagnostics/model_diagnostics_demo.ipynb
+++ b/examples/model_experiments/model_diagnostics/model_diagnostics_demo.ipynb
@@ -16,12 +16,12 @@
"metadata": {},
"source": [
"\n",
- "\n",
" | \n",
"\n",
"\n",
- "\n",
" | "
],
diff --git a/examples/model_diagnostics/model_diagnostics_guide.ipynb b/examples/model_experiments/model_diagnostics/model_diagnostics_guide.ipynb
similarity index 97%
rename from examples/model_diagnostics/model_diagnostics_guide.ipynb
rename to examples/model_experiments/model_diagnostics/model_diagnostics_guide.ipynb
index 9c355fcb1..4dae1cb8c 100644
--- a/examples/model_diagnostics/model_diagnostics_guide.ipynb
+++ b/examples/model_experiments/model_diagnostics/model_diagnostics_guide.ipynb
@@ -16,12 +16,12 @@
"metadata": {},
"source": [
"\n",
- "\n",
" | \n",
"\n",
"\n",
- "\n",
" | "
],
@@ -352,4 +352,4 @@
"execution_count": null
}
]
-}
+}
\ No newline at end of file
diff --git a/examples/prediction_upload/image_predictions.ipynb b/examples/prediction_upload/image_predictions.ipynb
index 593a0de10..7731d38a7 100644
--- a/examples/prediction_upload/image_predictions.ipynb
+++ b/examples/prediction_upload/image_predictions.ipynb
@@ -447,7 +447,6 @@
"mask_data = lb.types.MaskData(im_bytes=response.content) # You can also use \"url\" instead of img_bytes to pass the PNG mask url.\n",
"mask_prediction = lb_types.ObjectAnnotation(\n",
" name=\"mask\",\n",
- " confidence=0.5,\n",
" value=lb_types.Mask(\n",
" mask=mask_data,\n",
" color=(255, 255, 255))\n",
@@ -456,7 +455,6 @@
"# NDJSON using instanceURI, bytes array is not fully supported.\n",
"mask_prediction_ndjson = {\n",
" \"name\": \"mask\",\n",
- " \"confidence\": 0.5,\n",
" \"classifications\": [],\n",
" \"mask\": {\"instanceURI\": url,\n",
" \"colorRGB\": (255, 255, 255)}\n",
@@ -486,7 +484,6 @@
" name = \"mask_with_text_subclass\", # must match your ontology feature\"s name\n",
" value=lb_types.Mask(\n",
" mask=mask_data_2,\n",
- " confidence=0.5,\n",
" color=(255, 255, 255)),\n",
" classifications=[\n",
" lb_types.ClassificationAnnotation(\n",