Test benchmarker

tests/fixtures/unit_tests/benchmarker.py

@@ -0,0 +1,76 @@
+# Copyright (C) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+
+import pytest
+from pytest_mock import MockerFixture
+
+from geti_sdk.benchmarking import Benchmarker
+from geti_sdk.data_models.project import Project
+from geti_sdk.geti import Geti
+
+
+@pytest.fixture()
+def fxt_benchmarker(
+    mocker: MockerFixture,
+    fxt_classification_project: Project,
+    fxt_mocked_geti: Geti,
+) -> Benchmarker:
+    _ = mocker.patch(
+        "geti_sdk.geti.ProjectClient.get_project_by_name",
+        return_value=fxt_classification_project,
+    )
+    _ = mocker.patch("geti_sdk.benchmarking.benchmarker.ModelClient")
+    _ = mocker.patch("geti_sdk.benchmarking.benchmarker.TrainingClient")
+    project_name = "project name"
+    algorithms_to_benchmark = ("ALGO_1", "ALGO_2")
+    precision_levels = ("PRECISION_1", "PRECISION_2")
+    images = ("path_1", "path_2")
+    yield Benchmarker(
+        geti=fxt_mocked_geti,
+        project=project_name,
+        algorithms=algorithms_to_benchmark,
+        precision_levels=precision_levels,
+        benchmark_images=images,
+    )
+
+
+@pytest.fixture()
+def fxt_benchmarker_task_chain(
+    mocker: MockerFixture,
+    fxt_detection_to_classification_project: Project,
+    fxt_mocked_geti: Geti,
+) -> Benchmarker:
+    _ = mocker.patch(
+        "geti_sdk.geti.ProjectClient.get_project_by_name",
+        return_value=fxt_detection_to_classification_project,
+    )
+    model_client_object_mock = mocker.MagicMock()
+    _ = mocker.patch(
+        "geti_sdk.benchmarking.benchmarker.ModelClient",
+        return_value=model_client_object_mock,
+    )
+    active_models = (mocker.MagicMock(), mocker.MagicMock())
+    model_client_object_mock.get_all_active_models.return_value = active_models
+
+    _ = mocker.patch("geti_sdk.benchmarking.benchmarker.TrainingClient")
+    project_name = "project name"
+    precision_levels = ("PRECISION_1", "PRECISION_2")
+    images = ("path_1", "path_2")
+
+    yield Benchmarker(
+        geti=fxt_mocked_geti,
+        project=project_name,
+        precision_levels=precision_levels,
+        benchmark_images=images,
+    )

tests/fixtures/unit_tests/project.py

@@ -593,3 +593,10 @@ def fxt_nightly_projects(fxt_nightly_projects_rest) -> List[Project]:
 @pytest.fixture()
 def fxt_classification_project(fxt_nightly_projects: List[Project]) -> Project:
     yield fxt_nightly_projects[0]
+
+
+@pytest.fixture()
+def fxt_detection_to_classification_project(
+    fxt_nightly_projects: List[Project],
+) -> Project:
+    yield fxt_nightly_projects[2]

tests/nightly/test_classification.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2022 Intel Corporation
+# Copyright (C) 2024 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,14 +12,57 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
+import pandas as pd
 from test_nightly_project import TestNightlyProject
 
+from geti_sdk.benchmarking.benchmarker import Benchmarker
+from geti_sdk.geti import Geti
+from tests.helpers import project_service
+
 
 class TestClassification(TestNightlyProject):
     """
-    Class to test project creation, annotation upload, training, prediction and
+    Class to test project creation, annotation upload, training, prediction, benchmarking and
     deployment for a classification project
     """
 
     PROJECT_TYPE = "classification"
     __test__ = True
+
+    def test_benchmarking(
+        self,
+        fxt_project_service_no_vcr: project_service,
+        fxt_geti_no_vcr: Geti,
+        fxt_temp_directory: str,
+        fxt_image_path: str,
+        fxt_image_path_complex: str,
+    ):
+        """
+        Tests benchmarking for the project.
+        """
+        project = fxt_project_service_no_vcr.project
+        algorithms_to_benchmark = [
+            algo.algorithm_name
+            for algo in fxt_project_service_no_vcr._training_client.get_algorithms_for_task(
+                0
+            )
+        ][:2]
+        images = [fxt_image_path, fxt_image_path_complex]
+        precision_levels = ["FP16", "INT8"]
+
+        benchmarker = Benchmarker(
+            geti=fxt_geti_no_vcr,
+            project=project,
+            algorithms=algorithms_to_benchmark,
+            precision_levels=precision_levels,
+            benchmark_images=images,
+        )
+        benchmarker.prepare_benchmark(working_directory=fxt_temp_directory)
+        results = benchmarker.run_throughput_benchmark(
+            working_directory=fxt_temp_directory,
+            results_filename="results",
+            target_device="CPU",
+            frames=2,
+            repeats=2,
+        )
+        pd.DataFrame(results)

tests/nightly/test_detection_to_classification.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2022 Intel Corporation
+# Copyright (C) 2024 Intel Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
+import pandas as pd
 from test_nightly_project import TestNightlyProject
 
+from geti_sdk.benchmarking.benchmarker import Benchmarker
+from geti_sdk.geti import Geti
+from tests.helpers import project_service
+
 
 class TestDetectionToClassification(TestNightlyProject):
     """
@@ -23,3 +28,48 @@ class TestDetectionToClassification(TestNightlyProject):
 
     PROJECT_TYPE = "detection_to_classification"
     __test__ = True
+
+    def test_benchmarking(
+        self,
+        fxt_project_service_no_vcr: project_service,
+        fxt_geti_no_vcr: Geti,
+        fxt_temp_directory: str,
+        fxt_image_path: str,
+        fxt_image_path_complex: str,
+    ):
+        """
+        Tests benchmarking for the project.
+        """
+        project = fxt_project_service_no_vcr.project
+        images = [fxt_image_path, fxt_image_path_complex]
+        precision_levels = ["FP16", "INT8"]
+
+        benchmarker = Benchmarker(
+            geti=fxt_geti_no_vcr,
+            project=project,
+            precision_levels=precision_levels,
+            benchmark_images=images,
+        )
+        benchmarker.set_task_chain_algorithms(
+            [
+                algo.algorithm_name
+                for algo in fxt_project_service_no_vcr._training_client.get_algorithms_for_task(
+                    0
+                )
+            ][:2],
+            [
+                algo.algorithm_name
+                for algo in fxt_project_service_no_vcr._training_client.get_algorithms_for_task(
+                    1
+                )
+            ][:2],
+        )
+        benchmarker.prepare_benchmark(working_directory=fxt_temp_directory)
+        results = benchmarker.run_throughput_benchmark(
+            working_directory=fxt_temp_directory,
+            results_filename="results",
+            target_device="CPU",
+            frames=2,
+            repeats=2,
+        )
+        pd.DataFrame(results)