From e5949bac754d301d69ae32281556e8c0320784ac Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Tue, 11 Mar 2025 11:16:57 -0500 Subject: [PATCH 01/11] Add a Detection union block --- inference/core/workflows/core_steps/loader.py | 4 + .../detections_union/__init__.py | 0 .../transformations/detections_union/v1.py | 118 ++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 inference/core/workflows/core_steps/transformations/detections_union/__init__.py create mode 100644 inference/core/workflows/core_steps/transformations/detections_union/v1.py diff --git a/inference/core/workflows/core_steps/loader.py b/inference/core/workflows/core_steps/loader.py index 7b6f8dcb61..41b5d2b363 100644 --- a/inference/core/workflows/core_steps/loader.py +++ b/inference/core/workflows/core_steps/loader.py @@ -301,6 +301,9 @@ from inference.core.workflows.core_steps.transformations.detections_transformation.v1 import ( DetectionsTransformationBlockV1, ) +from inference.core.workflows.core_steps.transformations.detections_union.v1 import ( + DetectionsUnionBlockV1, +) from inference.core.workflows.core_steps.transformations.dynamic_crop.v1 import ( DynamicCropBlockV1, ) @@ -524,6 +527,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: BlurVisualizationBlockV1, BoundingBoxVisualizationBlockV1, BoundingRectBlockV1, + DetectionsUnionBlockV1, ByteTrackerBlockV2, CacheGetBlockV1, CacheSetBlockV1, diff --git a/inference/core/workflows/core_steps/transformations/detections_union/__init__.py b/inference/core/workflows/core_steps/transformations/detections_union/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/inference/core/workflows/core_steps/transformations/detections_union/v1.py b/inference/core/workflows/core_steps/transformations/detections_union/v1.py new file mode 100644 index 0000000000..db92220ca9 --- /dev/null +++ b/inference/core/workflows/core_steps/transformations/detections_union/v1.py @@ -0,0 +1,118 @@ +from typing import Any, Dict, List, Literal, Optional, Type + +import numpy as np +import supervision as sv +from pydantic import ConfigDict, Field + +from inference.core.workflows.execution_engine.entities.base import OutputDefinition +from inference.core.workflows.execution_engine.entities.types import ( + OBJECT_DETECTION_PREDICTION_KIND, + Selector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + +OUTPUT_KEY: str = "predictions" + +SHORT_DESCRIPTION = "Merge multiple detections into a single bounding box." +LONG_DESCRIPTION = """ +The `DetectionsUnion` block combines multiple detections into a single bounding box that encompasses all input detections. +This is useful when you want to: +- Merge overlapping or nearby detections of the same object +- Create a single region that contains multiple detected objects +- Simplify multiple detections into one larger detection + +The resulting detection will have: +- A bounding box that contains all input detections +- The class_id and confidence from the first detection in the input +""" + + +class DetectionsUnionManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Detections Union", + "version": "v1", + "short_description": SHORT_DESCRIPTION, + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "transformation", + "ui_manifest": { + "section": "transformation", + "icon": "fal fa-object-union", + "blockPriority": 5, + }, + } + ) + type: Literal["roboflow_core/detections_union@v1"] + predictions: Selector( + kind=[ + OBJECT_DETECTION_PREDICTION_KIND, + ] + ) = Field( + description="Object detection predictions to merge into a single bounding box.", + examples=["$steps.object_detection_model.predictions"], + ) + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name=OUTPUT_KEY, kind=[OBJECT_DETECTION_PREDICTION_KIND]), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + +def calculate_union_bbox(detections: sv.Detections) -> np.ndarray: + """Calculate a single bounding box that contains all input detections.""" + if len(detections) == 0: + return np.array([]) + + # Get all bounding boxes + xyxy = detections.xyxy + + # Calculate the union by taking min/max coordinates + x1 = np.min(xyxy[:, 0]) + y1 = np.min(xyxy[:, 1]) + x2 = np.max(xyxy[:, 2]) + y2 = np.max(xyxy[:, 3]) + + return np.array([[x1, y1, x2, y2]]) + + +class DetectionsUnionBlockV1(WorkflowBlock): + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return DetectionsUnionManifest + + def run( + self, + predictions: sv.Detections, + ) -> BlockResult: + if predictions is None or len(predictions) == 0: + return {OUTPUT_KEY: sv.Detections(xyxy=np.array([]))} + + # Calculate the union bounding box + union_bbox = calculate_union_bbox(predictions) + + # Create a new detection with the union bbox and ensure numpy arrays for all fields + merged_detection = sv.Detections( + xyxy=union_bbox, + confidence=( + np.array([predictions.confidence[0]], dtype=np.float32) + if predictions.confidence is not None + else None + ), + class_id=( + np.array([predictions.class_id[0]], dtype=np.int32) + if predictions.class_id is not None + else None + ), + ) + + return {OUTPUT_KEY: merged_detection} From f74418a90ccc955ebaf46a6de33a28eed16426d5 Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Tue, 11 Mar 2025 13:10:03 -0500 Subject: [PATCH 02/11] fix serialization issue --- .../core_steps/transformations/detections_union/v1.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/inference/core/workflows/core_steps/transformations/detections_union/v1.py b/inference/core/workflows/core_steps/transformations/detections_union/v1.py index db92220ca9..8707e95c5a 100644 --- a/inference/core/workflows/core_steps/transformations/detections_union/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_union/v1.py @@ -1,4 +1,5 @@ from typing import Any, Dict, List, Literal, Optional, Type +from uuid import uuid4 import numpy as np import supervision as sv @@ -113,6 +114,10 @@ def run( if predictions.class_id is not None else None ), + data={ + "class_name": np.array([predictions.data["class_name"][0]]), + "detection_id": np.array([str(uuid4())]), + }, ) return {OUTPUT_KEY: merged_detection} From 7c11460c94009991a39d6e2288d4509b7486347f Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Thu, 13 Mar 2025 14:44:26 -0500 Subject: [PATCH 03/11] rename block to detections merge --- inference/core/workflows/core_steps/loader.py | 6 +++--- .../__init__.py | 0 .../v1.py | 19 ++++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) rename inference/core/workflows/core_steps/transformations/{detections_union => detections_merge}/__init__.py (100%) rename inference/core/workflows/core_steps/transformations/{detections_union => detections_merge}/v1.py (87%) diff --git a/inference/core/workflows/core_steps/loader.py b/inference/core/workflows/core_steps/loader.py index 41b5d2b363..aa320e4141 100644 --- a/inference/core/workflows/core_steps/loader.py +++ b/inference/core/workflows/core_steps/loader.py @@ -301,8 +301,8 @@ from inference.core.workflows.core_steps.transformations.detections_transformation.v1 import ( DetectionsTransformationBlockV1, ) -from inference.core.workflows.core_steps.transformations.detections_union.v1 import ( - DetectionsUnionBlockV1, +from inference.core.workflows.core_steps.transformations.detections_merge.v1 import ( + DetectionsMergeBlockV1, ) from inference.core.workflows.core_steps.transformations.dynamic_crop.v1 import ( DynamicCropBlockV1, @@ -527,7 +527,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: BlurVisualizationBlockV1, BoundingBoxVisualizationBlockV1, BoundingRectBlockV1, - DetectionsUnionBlockV1, + DetectionsMergeBlockV1, ByteTrackerBlockV2, CacheGetBlockV1, CacheSetBlockV1, diff --git a/inference/core/workflows/core_steps/transformations/detections_union/__init__.py b/inference/core/workflows/core_steps/transformations/detections_merge/__init__.py similarity index 100% rename from inference/core/workflows/core_steps/transformations/detections_union/__init__.py rename to inference/core/workflows/core_steps/transformations/detections_merge/__init__.py diff --git a/inference/core/workflows/core_steps/transformations/detections_union/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py similarity index 87% rename from inference/core/workflows/core_steps/transformations/detections_union/v1.py rename to inference/core/workflows/core_steps/transformations/detections_merge/v1.py index 8707e95c5a..382b14fd92 100644 --- a/inference/core/workflows/core_steps/transformations/detections_union/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -7,7 +7,9 @@ from inference.core.workflows.execution_engine.entities.base import OutputDefinition from inference.core.workflows.execution_engine.entities.types import ( - OBJECT_DETECTION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + KEYPOINT_DETECTION_PREDICTION_KIND, Selector, ) from inference.core.workflows.prototypes.block import ( @@ -20,7 +22,7 @@ SHORT_DESCRIPTION = "Merge multiple detections into a single bounding box." LONG_DESCRIPTION = """ -The `DetectionsUnion` block combines multiple detections into a single bounding box that encompasses all input detections. +The `DetectionsMerge` block combines multiple detections into a single bounding box that encompasses all input detections. This is useful when you want to: - Merge overlapping or nearby detections of the same object - Create a single region that contains multiple detected objects @@ -32,10 +34,10 @@ """ -class DetectionsUnionManifest(WorkflowBlockManifest): +class DetectionsMergeManifest(WorkflowBlockManifest): model_config = ConfigDict( json_schema_extra={ - "name": "Detections Union", + "name": "Detections Merge", "version": "v1", "short_description": SHORT_DESCRIPTION, "long_description": LONG_DESCRIPTION, @@ -48,10 +50,13 @@ class DetectionsUnionManifest(WorkflowBlockManifest): }, } ) - type: Literal["roboflow_core/detections_union@v1"] + type: Literal["roboflow_core/detections_merge@v1"] predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + KEYPOINT_DETECTION_PREDICTION_KIND, + ] ) = Field( description="Object detection predictions to merge into a single bounding box.", @@ -86,10 +91,10 @@ def calculate_union_bbox(detections: sv.Detections) -> np.ndarray: return np.array([[x1, y1, x2, y2]]) -class DetectionsUnionBlockV1(WorkflowBlock): +class DetectionsMergeBlockV1(WorkflowBlock): @classmethod def get_manifest(cls) -> Type[WorkflowBlockManifest]: - return DetectionsUnionManifest + return DetectionsMergeManifest def run( self, From b8802f08846d755a7ff042fe77cff7920c04ea0e Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Thu, 13 Mar 2025 14:57:43 -0500 Subject: [PATCH 04/11] fix empty case and add unit tests --- .../transformations/detections_merge/v1.py | 5 +- .../transformations/test_detections_merge.py | 88 +++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py diff --git a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py index 382b14fd92..94942d69d7 100644 --- a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -56,7 +56,6 @@ class DetectionsMergeManifest(WorkflowBlockManifest): OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, - ] ) = Field( description="Object detection predictions to merge into a single bounding box.", @@ -77,7 +76,7 @@ def get_execution_engine_compatibility(cls) -> Optional[str]: def calculate_union_bbox(detections: sv.Detections) -> np.ndarray: """Calculate a single bounding box that contains all input detections.""" if len(detections) == 0: - return np.array([]) + return np.array([], dtype=np.float32).reshape(0, 4) # Get all bounding boxes xyxy = detections.xyxy @@ -101,7 +100,7 @@ def run( predictions: sv.Detections, ) -> BlockResult: if predictions is None or len(predictions) == 0: - return {OUTPUT_KEY: sv.Detections(xyxy=np.array([]))} + return {OUTPUT_KEY: sv.Detections(xyxy=np.array([], dtype=np.float32).reshape(0, 4))} # Calculate the union bounding box union_bbox = calculate_union_bbox(predictions) diff --git a/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py b/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py new file mode 100644 index 0000000000..4dfeaa20d4 --- /dev/null +++ b/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py @@ -0,0 +1,88 @@ +import numpy as np +import pytest +import supervision as sv + +from inference.core.workflows.core_steps.transformations.detections_merge.v1 import ( + DetectionsMergeBlockV1, + DetectionsMergeManifest, + calculate_union_bbox, +) + + +def test_calculate_union_bbox(): + # given + detections = sv.Detections( + xyxy=np.array([[10, 10, 20, 20], [15, 15, 25, 25]]), + ) + + # when + union_bbox = calculate_union_bbox(detections) + + # then + expected_bbox = np.array([[10, 10, 25, 25]]) + assert np.allclose( + union_bbox, expected_bbox + ), f"Expected bounding box to be {expected_bbox}, but got {union_bbox}" + + +@pytest.mark.parametrize("type_alias", ["roboflow_core/detections_merge@v1"]) +def test_detections_merge_validation_when_valid_manifest_is_given( + type_alias: str, +) -> None: + # given + data = { + "type": type_alias, + "name": "detections_merge", + "predictions": "$steps.od_model.predictions", + } + + # when + result = DetectionsMergeManifest.model_validate(data) + + # then + assert result == DetectionsMergeManifest( + type=type_alias, name="detections_merge", predictions="$steps.od_model.predictions" + ) + + +def test_detections_merge_block() -> None: + # given + block = DetectionsMergeBlockV1() + detections = sv.Detections( + xyxy=np.array([[10, 10, 20, 20], [15, 15, 25, 25]]), + confidence=np.array([0.9, 0.8]), + class_id=np.array([1, 1]), + data={ + "class_name": np.array(["person", "person"]), + }, + ) + + # when + output = block.run(predictions=detections) + + # then + assert isinstance(output, dict) + assert "predictions" in output + assert len(output["predictions"]) == 1 + assert np.allclose(output["predictions"].xyxy, np.array([[10, 10, 25, 25]])) + assert np.allclose(output["predictions"].confidence, np.array([0.9])) + assert np.allclose(output["predictions"].class_id, np.array([1])) + assert output["predictions"].data["class_name"][0] == "person" + assert "detection_id" in output["predictions"].data + assert len(output["predictions"].data["detection_id"]) == 1 + + +def test_detections_merge_block_empty_input() -> None: + # given + block = DetectionsMergeBlockV1() + empty_detections = sv.Detections(xyxy=np.array([], dtype=np.float32).reshape(0, 4)) + + # when + output = block.run(predictions=empty_detections) + + # then + assert isinstance(output, dict) + assert "predictions" in output + assert len(output["predictions"]) == 0 + assert isinstance(output["predictions"].xyxy, np.ndarray) + assert output["predictions"].xyxy.shape == (0, 4) \ No newline at end of file From bdb81341a3455c93bf83e9f00aef7b9831089a67 Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Thu, 13 Mar 2025 15:01:13 -0500 Subject: [PATCH 05/11] add workflow test for detections merge --- .../test_workflow_with_detections_merge.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 tests/workflows/integration_tests/execution/test_workflow_with_detections_merge.py diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_detections_merge.py b/tests/workflows/integration_tests/execution/test_workflow_with_detections_merge.py new file mode 100644 index 0000000000..c0f5b6f274 --- /dev/null +++ b/tests/workflows/integration_tests/execution/test_workflow_with_detections_merge.py @@ -0,0 +1,109 @@ +import numpy as np +import pytest +import supervision as sv + +from inference.core.env import WORKFLOWS_MAX_CONCURRENT_STEPS +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.execution_engine.core import ExecutionEngine +from tests.workflows.integration_tests.execution.workflows_gallery_collector.decorators import ( + add_to_workflows_gallery, +) + +DETECTIONS_MERGE_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + ], + "steps": [ + { + "type": "ObjectDetectionModel", + "name": "detection", + "image": "$inputs.image", + "model_id": "yolov8n-640", + }, + { + "type": "roboflow_core/detections_merge@v1", + "name": "detections_merge", + "predictions": "$steps.detection.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.detections_merge.predictions", + } + ], +} + + +@add_to_workflows_gallery( + category="Basic Workflows", + use_case_title="Workflow with detections merge", + use_case_description=""" +This workflow demonstrates how to merge multiple object detections into a single bounding box. +This is useful when you want to: +- Combine overlapping detections of the same object +- Create a single region that contains multiple detected objects +- Simplify multiple detections into one larger detection + """, + workflow_definition=DETECTIONS_MERGE_WORKFLOW, + workflow_name_in_app="merge-detections", +) +def test_detections_merge_workflow( + model_manager: ModelManager, + dogs_image: np.ndarray, +) -> None: + # given + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": None, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + } + execution_engine = ExecutionEngine.init( + workflow_definition=DETECTIONS_MERGE_WORKFLOW, + init_parameters=workflow_init_parameters, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + + # when + result = execution_engine.run( + runtime_parameters={ + "image": [dogs_image], + } + ) + + # then + assert len(result) == 1, "One set of outputs expected" + assert "result" in result[0], "Output must contain key 'result'" + assert isinstance( + result[0]["result"], sv.Detections + ), "Output must be instance of sv.Detections" + + # Check that we have exactly one merged detection + assert len(result[0]["result"]) == 1, "Should have exactly one merged detection" + + # Check that the merged detection has all required fields + assert "class_name" in result[0]["result"].data, "Should have class_name in data" + assert "detection_id" in result[0]["result"].data, "Should have detection_id in data" + + # Check that the bounding box has reasonable dimensions + merged_bbox = result[0]["result"].xyxy[0] + image_height, image_width = dogs_image.shape[:2] + + # Check that coordinates are within image bounds + assert 0 <= merged_bbox[0] <= image_width, "x1 should be within image bounds" + assert 0 <= merged_bbox[1] <= image_height, "y1 should be within image bounds" + assert 0 <= merged_bbox[2] <= image_width, "x2 should be within image bounds" + assert 0 <= merged_bbox[3] <= image_height, "y2 should be within image bounds" + + # Check that the box has reasonable dimensions + assert merged_bbox[2] > merged_bbox[0], "x2 should be greater than x1" + assert merged_bbox[3] > merged_bbox[1], "y2 should be greater than y1" + + # Check that the box is large enough to likely contain the dogs + box_width = merged_bbox[2] - merged_bbox[0] + box_height = merged_bbox[3] - merged_bbox[1] + assert box_width > 100, "Merged box should be reasonably wide" + assert box_height > 100, "Merged box should be reasonably tall" \ No newline at end of file From fb898a433fdac458c37c33c3da4fe759cd2564ca Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Thu, 13 Mar 2025 15:05:18 -0500 Subject: [PATCH 06/11] make style --- cpu_http.py | 51 +++++++++++++++++++ inference/core/workflows/core_steps/loader.py | 6 +-- .../transformations/detections_merge/v1.py | 8 ++- 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 cpu_http.py diff --git a/cpu_http.py b/cpu_http.py new file mode 100644 index 0000000000..86184b34ba --- /dev/null +++ b/cpu_http.py @@ -0,0 +1,51 @@ +from functools import partial +from multiprocessing import Process + +from inference.core.cache import cache +from inference.core.interfaces.http.http_api import HttpInterface +from inference.core.interfaces.stream_manager.manager_app.app import start +from inference.core.managers.active_learning import ( + ActiveLearningManager, + BackgroundTaskActiveLearningManager, +) +from inference.core.managers.base import ModelManager +from inference.core.managers.decorators.fixed_size_cache import WithFixedSizeCache +from inference.core.registries.roboflow import ( + RoboflowModelRegistry, +) + +from inference.core.env import ( + MAX_ACTIVE_MODELS, + ACTIVE_LEARNING_ENABLED, + GCP_SERVERLESS, + LAMBDA, + ENABLE_STREAM_API, + STREAM_API_PRELOADED_PROCESSES, +) +from inference.models.utils import ROBOFLOW_MODEL_TYPES + + +if ENABLE_STREAM_API: + stream_manager_process = Process( + target=partial(start, expected_warmed_up_pipelines=STREAM_API_PRELOADED_PROCESSES), + ) + stream_manager_process.start() + +model_registry = RoboflowModelRegistry(ROBOFLOW_MODEL_TYPES) + +if ACTIVE_LEARNING_ENABLED: + if LAMBDA or GCP_SERVERLESS: + model_manager = ActiveLearningManager( + model_registry=model_registry, cache=cache + ) + else: + model_manager = BackgroundTaskActiveLearningManager( + model_registry=model_registry, cache=cache + ) +else: + model_manager = ModelManager(model_registry=model_registry) + +model_manager = WithFixedSizeCache(model_manager, max_size=MAX_ACTIVE_MODELS) +model_manager.init_pingback() +interface = HttpInterface(model_manager) +app = interface.app diff --git a/inference/core/workflows/core_steps/loader.py b/inference/core/workflows/core_steps/loader.py index aa320e4141..bfa3f64e97 100644 --- a/inference/core/workflows/core_steps/loader.py +++ b/inference/core/workflows/core_steps/loader.py @@ -298,12 +298,12 @@ from inference.core.workflows.core_steps.transformations.detections_filter.v1 import ( DetectionsFilterBlockV1, ) -from inference.core.workflows.core_steps.transformations.detections_transformation.v1 import ( - DetectionsTransformationBlockV1, -) from inference.core.workflows.core_steps.transformations.detections_merge.v1 import ( DetectionsMergeBlockV1, ) +from inference.core.workflows.core_steps.transformations.detections_transformation.v1 import ( + DetectionsTransformationBlockV1, +) from inference.core.workflows.core_steps.transformations.dynamic_crop.v1 import ( DynamicCropBlockV1, ) diff --git a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py index 94942d69d7..05915234aa 100644 --- a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -7,9 +7,9 @@ from inference.core.workflows.execution_engine.entities.base import OutputDefinition from inference.core.workflows.execution_engine.entities.types import ( - OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, Selector, ) from inference.core.workflows.prototypes.block import ( @@ -100,7 +100,11 @@ def run( predictions: sv.Detections, ) -> BlockResult: if predictions is None or len(predictions) == 0: - return {OUTPUT_KEY: sv.Detections(xyxy=np.array([], dtype=np.float32).reshape(0, 4))} + return { + OUTPUT_KEY: sv.Detections( + xyxy=np.array([], dtype=np.float32).reshape(0, 4) + ) + } # Calculate the union bounding box union_bbox = calculate_union_bbox(predictions) From a13e3877a64507fd1f4a799eead848b2d8a8812f Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Thu, 13 Mar 2025 15:06:21 -0500 Subject: [PATCH 07/11] remove accidentally comited file --- cpu_http.py | 51 --------------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 cpu_http.py diff --git a/cpu_http.py b/cpu_http.py deleted file mode 100644 index 86184b34ba..0000000000 --- a/cpu_http.py +++ /dev/null @@ -1,51 +0,0 @@ -from functools import partial -from multiprocessing import Process - -from inference.core.cache import cache -from inference.core.interfaces.http.http_api import HttpInterface -from inference.core.interfaces.stream_manager.manager_app.app import start -from inference.core.managers.active_learning import ( - ActiveLearningManager, - BackgroundTaskActiveLearningManager, -) -from inference.core.managers.base import ModelManager -from inference.core.managers.decorators.fixed_size_cache import WithFixedSizeCache -from inference.core.registries.roboflow import ( - RoboflowModelRegistry, -) - -from inference.core.env import ( - MAX_ACTIVE_MODELS, - ACTIVE_LEARNING_ENABLED, - GCP_SERVERLESS, - LAMBDA, - ENABLE_STREAM_API, - STREAM_API_PRELOADED_PROCESSES, -) -from inference.models.utils import ROBOFLOW_MODEL_TYPES - - -if ENABLE_STREAM_API: - stream_manager_process = Process( - target=partial(start, expected_warmed_up_pipelines=STREAM_API_PRELOADED_PROCESSES), - ) - stream_manager_process.start() - -model_registry = RoboflowModelRegistry(ROBOFLOW_MODEL_TYPES) - -if ACTIVE_LEARNING_ENABLED: - if LAMBDA or GCP_SERVERLESS: - model_manager = ActiveLearningManager( - model_registry=model_registry, cache=cache - ) - else: - model_manager = BackgroundTaskActiveLearningManager( - model_registry=model_registry, cache=cache - ) -else: - model_manager = ModelManager(model_registry=model_registry) - -model_manager = WithFixedSizeCache(model_manager, max_size=MAX_ACTIVE_MODELS) -model_manager.init_pingback() -interface = HttpInterface(model_manager) -app = interface.app From cdfa960554a2634f4d8b1331d5d91699d0bab14c Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Fri, 14 Mar 2025 09:19:31 -0500 Subject: [PATCH 08/11] change to use lowest confidence value for merged prediction --- .../transformations/detections_merge/v1.py | 15 ++++++++++++--- .../transformations/test_detections_merge.py | 5 ++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py index 05915234aa..e16d9e7e0a 100644 --- a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -31,6 +31,7 @@ The resulting detection will have: - A bounding box that contains all input detections - The class_id and confidence from the first detection in the input +- The confidence is set to the lowest confidence among all detections """ @@ -54,8 +55,6 @@ class DetectionsMergeManifest(WorkflowBlockManifest): predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, - INSTANCE_SEGMENTATION_PREDICTION_KIND, - KEYPOINT_DETECTION_PREDICTION_KIND, ] ) = Field( description="Object detection predictions to merge into a single bounding box.", @@ -90,6 +89,13 @@ def calculate_union_bbox(detections: sv.Detections) -> np.ndarray: return np.array([[x1, y1, x2, y2]]) +def get_lowest_confidence_index(detections: sv.Detections) -> int: + """Get the index of the detection with the lowest confidence.""" + if detections.confidence is None: + return 0 + return int(np.argmin(detections.confidence)) + + class DetectionsMergeBlockV1(WorkflowBlock): @classmethod def get_manifest(cls) -> Type[WorkflowBlockManifest]: @@ -109,11 +115,14 @@ def run( # Calculate the union bounding box union_bbox = calculate_union_bbox(predictions) + # Get the index of the detection with lowest confidence + lowest_conf_idx = get_lowest_confidence_index(predictions) + # Create a new detection with the union bbox and ensure numpy arrays for all fields merged_detection = sv.Detections( xyxy=union_bbox, confidence=( - np.array([predictions.confidence[0]], dtype=np.float32) + np.array([predictions.confidence[lowest_conf_idx]], dtype=np.float32) if predictions.confidence is not None else None ), diff --git a/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py b/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py index 4dfeaa20d4..f114b29319 100644 --- a/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py +++ b/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py @@ -65,11 +65,10 @@ def test_detections_merge_block() -> None: assert "predictions" in output assert len(output["predictions"]) == 1 assert np.allclose(output["predictions"].xyxy, np.array([[10, 10, 25, 25]])) - assert np.allclose(output["predictions"].confidence, np.array([0.9])) + assert np.allclose(output["predictions"].confidence, np.array([0.8])) assert np.allclose(output["predictions"].class_id, np.array([1])) assert output["predictions"].data["class_name"][0] == "person" - assert "detection_id" in output["predictions"].data - assert len(output["predictions"].data["detection_id"]) == 1 + assert isinstance(output["predictions"].data["detection_id"][0], str) def test_detections_merge_block_empty_input() -> None: From 9ea259295810f295768a198bb74baa889c706743 Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Fri, 14 Mar 2025 09:20:27 -0500 Subject: [PATCH 09/11] allow instance seg and keypoint inputs --- .../workflows/core_steps/transformations/detections_merge/v1.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py index e16d9e7e0a..642a997ac7 100644 --- a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -55,6 +55,8 @@ class DetectionsMergeManifest(WorkflowBlockManifest): predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + KEYPOINT_DETECTION_PREDICTION_KIND, ] ) = Field( description="Object detection predictions to merge into a single bounding box.", From 88cba26c84669192accf8b1c62c4827d58d677a2 Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Fri, 14 Mar 2025 09:28:59 -0500 Subject: [PATCH 10/11] use a fixed classname for merged detection (default "merged_detection", can be user specified) --- .../transformations/detections_merge/v1.py | 15 ++++---- .../transformations/test_detections_merge.py | 36 +++++++++++++++++-- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py index 642a997ac7..93bd711b3a 100644 --- a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -30,7 +30,7 @@ The resulting detection will have: - A bounding box that contains all input detections -- The class_id and confidence from the first detection in the input +- The classname of the merged detection is set to "merged_detection" by default, but can be customized via the `class_name` parameter - The confidence is set to the lowest confidence among all detections """ @@ -62,6 +62,10 @@ class DetectionsMergeManifest(WorkflowBlockManifest): description="Object detection predictions to merge into a single bounding box.", examples=["$steps.object_detection_model.predictions"], ) + class_name: str = Field( + default="merged_detection", + description="The class name to assign to the merged detection.", + ) @classmethod def describe_outputs(cls) -> List[OutputDefinition]: @@ -106,6 +110,7 @@ def get_manifest(cls) -> Type[WorkflowBlockManifest]: def run( self, predictions: sv.Detections, + class_name: str = "merged_detection", ) -> BlockResult: if predictions is None or len(predictions) == 0: return { @@ -128,13 +133,9 @@ def run( if predictions.confidence is not None else None ), - class_id=( - np.array([predictions.class_id[0]], dtype=np.int32) - if predictions.class_id is not None - else None - ), + class_id=np.array([0], dtype=np.int32), # Fixed class_id of 0 for merged detection data={ - "class_name": np.array([predictions.data["class_name"][0]]), + "class_name": np.array([class_name]), "detection_id": np.array([str(uuid4())]), }, ) diff --git a/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py b/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py index f114b29319..f9683938ce 100644 --- a/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py +++ b/tests/workflows/unit_tests/core_steps/transformations/test_detections_merge.py @@ -34,6 +34,7 @@ def test_detections_merge_validation_when_valid_manifest_is_given( "type": type_alias, "name": "detections_merge", "predictions": "$steps.od_model.predictions", + "class_name": "custom_merged", } # when @@ -41,7 +42,10 @@ def test_detections_merge_validation_when_valid_manifest_is_given( # then assert result == DetectionsMergeManifest( - type=type_alias, name="detections_merge", predictions="$steps.od_model.predictions" + type=type_alias, + name="detections_merge", + predictions="$steps.od_model.predictions", + class_name="custom_merged", ) @@ -66,8 +70,34 @@ def test_detections_merge_block() -> None: assert len(output["predictions"]) == 1 assert np.allclose(output["predictions"].xyxy, np.array([[10, 10, 25, 25]])) assert np.allclose(output["predictions"].confidence, np.array([0.8])) - assert np.allclose(output["predictions"].class_id, np.array([1])) - assert output["predictions"].data["class_name"][0] == "person" + assert np.allclose(output["predictions"].class_id, np.array([0])) + assert output["predictions"].data["class_name"][0] == "merged_detection" + assert isinstance(output["predictions"].data["detection_id"][0], str) + + +def test_detections_merge_block_with_custom_class() -> None: + # given + block = DetectionsMergeBlockV1() + detections = sv.Detections( + xyxy=np.array([[10, 10, 20, 20], [15, 15, 25, 25]]), + confidence=np.array([0.9, 0.8]), + class_id=np.array([1, 1]), + data={ + "class_name": np.array(["person", "person"]), + }, + ) + + # when + output = block.run(predictions=detections, class_name="custom_merged") + + # then + assert isinstance(output, dict) + assert "predictions" in output + assert len(output["predictions"]) == 1 + assert np.allclose(output["predictions"].xyxy, np.array([[10, 10, 25, 25]])) + assert np.allclose(output["predictions"].confidence, np.array([0.8])) + assert np.allclose(output["predictions"].class_id, np.array([0])) + assert output["predictions"].data["class_name"][0] == "custom_merged" assert isinstance(output["predictions"].data["detection_id"][0], str) From 5ead88c53c8a23859367bce6816c5954423eb48d Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Fri, 14 Mar 2025 09:30:43 -0500 Subject: [PATCH 11/11] make style --- .../core_steps/transformations/detections_merge/v1.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py index 93bd711b3a..0292968ff8 100644 --- a/inference/core/workflows/core_steps/transformations/detections_merge/v1.py +++ b/inference/core/workflows/core_steps/transformations/detections_merge/v1.py @@ -133,7 +133,9 @@ def run( if predictions.confidence is not None else None ), - class_id=np.array([0], dtype=np.int32), # Fixed class_id of 0 for merged detection + class_id=np.array( + [0], dtype=np.int32 + ), # Fixed class_id of 0 for merged detection data={ "class_name": np.array([class_name]), "detection_id": np.array([str(uuid4())]),