diff --git a/tests/fixtures/unit_tests/benchmarker.py b/tests/fixtures/unit_tests/benchmarker.py new file mode 100644 index 00000000..6305c4a5 --- /dev/null +++ b/tests/fixtures/unit_tests/benchmarker.py @@ -0,0 +1,81 @@ +# Copyright (C) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import pytest +from pytest_mock import MockerFixture + +from geti_sdk.benchmarking import Benchmarker +from geti_sdk.data_models.project import Project +from geti_sdk.geti import Geti + + +@pytest.fixture() +def fxt_benchmarker( + mocker: MockerFixture, + fxt_classification_project: Project, + fxt_mocked_geti: Geti, +) -> Benchmarker: + _ = mocker.patch( + "geti_sdk.geti.ProjectClient.get_project_by_name", + return_value=fxt_classification_project, + ) + _ = mocker.patch( + "geti_sdk.benchmarking.benchmarker.ModelClient" + ) + _ = mocker.patch( + "geti_sdk.benchmarking.benchmarker.TrainingClient" + ) + project_name = "project name" + algorithms_to_benchmark = ("ALGO_1", "ALGO_2") + precision_levels = ("PRECISION_1", "PRECISION_2") + images = ("path_1", "path_2") + yield Benchmarker( + geti=fxt_mocked_geti, + project=project_name, + algorithms=algorithms_to_benchmark, + precision_levels=precision_levels, + benchmark_images=images, + ) + + +@pytest.fixture() +def fxt_benchmarker_task_chain( + mocker: MockerFixture, + fxt_detection_to_classification_project: Project, + fxt_mocked_geti: Geti, +) -> Benchmarker: + _ = mocker.patch( + "geti_sdk.geti.ProjectClient.get_project_by_name", + return_value=fxt_detection_to_classification_project, + ) + model_client_object_mock = mocker.MagicMock() + _ = mocker.patch( + "geti_sdk.benchmarking.benchmarker.ModelClient", return_value=model_client_object_mock + ) + active_models = (mocker.MagicMock(), mocker.MagicMock()) + model_client_object_mock.get_all_active_models.return_value = active_models + + _ = mocker.patch( + "geti_sdk.benchmarking.benchmarker.TrainingClient" + ) + project_name = "project name" + precision_levels = ("PRECISION_1", "PRECISION_2") + images = ("path_1", "path_2") + + yield Benchmarker( + geti=fxt_mocked_geti, + project=project_name, + precision_levels=precision_levels, + benchmark_images=images, + ) diff --git a/tests/fixtures/unit_tests/project.py b/tests/fixtures/unit_tests/project.py index f3fde4f6..84fa66c2 100644 --- a/tests/fixtures/unit_tests/project.py +++ b/tests/fixtures/unit_tests/project.py @@ -593,3 +593,10 @@ def fxt_nightly_projects(fxt_nightly_projects_rest) -> List[Project]: @pytest.fixture() def fxt_classification_project(fxt_nightly_projects: List[Project]) -> Project: yield fxt_nightly_projects[0] + + +@pytest.fixture() +def fxt_detection_to_classification_project( + fxt_nightly_projects: List[Project] +) -> Project: + yield fxt_nightly_projects[2] diff --git a/tests/pre-merge/unit/benchmarking/test_benchmarker.py b/tests/pre-merge/unit/benchmarking/test_benchmarker.py new file mode 100644 index 00000000..e0de2b47 --- /dev/null +++ b/tests/pre-merge/unit/benchmarking/test_benchmarker.py @@ -0,0 +1,272 @@ +# Copyright (C) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions +# and limitations under the License. + +import csv +from pathlib import Path + +import pytest +from pytest_mock import MockerFixture + +from geti_sdk import Geti +from geti_sdk.benchmarking import Benchmarker +from geti_sdk.data_models import Project + + +class TestBenchmarker: + def test_initialize( + self, + fxt_mocked_geti: Geti, + fxt_classification_project: Project, + mocker: MockerFixture + ): + # Arrange + mock_get_project_by_name = mocker.patch( + "geti_sdk.geti.ProjectClient.get_project_by_name", + return_value=fxt_classification_project, + ) + mocked_model_client = mocker.patch( + "geti_sdk.benchmarking.benchmarker.ModelClient" + ) + mocked_training_client = mocker.patch( + "geti_sdk.benchmarking.benchmarker.TrainingClient" + ) + project_name = "project name" + algorithms_to_benchmark = ("ALGO_1", "ALGO_2") + precision_levels = ("PRECISION_1", "PRECISION_2") + images = ("path_1", "path_2") + videos = ("path_3", "path_4") + + # Act + # Single task project, benchmarking on images + benchmarker = Benchmarker( + geti=fxt_mocked_geti, + project=project_name, + algorithms=algorithms_to_benchmark, + precision_levels=precision_levels, + benchmark_images=images, + ) + + # Assert + mock_get_project_by_name.assert_called_once_with(project_name=project_name) + mocked_model_client.assert_called_once() + mocked_training_client.assert_called_once() + assert benchmarker._is_single_task + assert not benchmarker._are_models_specified + assert benchmarker.precision_levels == precision_levels + assert benchmarker._algorithms == algorithms_to_benchmark + + # Act 2 + # Single task project, videos and images + with pytest.raises(ValueError): + benchmarker = Benchmarker( + geti=fxt_mocked_geti, + project=project_name, + algorithms=algorithms_to_benchmark, + precision_levels=precision_levels, + benchmark_images=images, + benchmark_video=videos + ) + + def test_initialize_task_chain( + self, + fxt_mocked_geti: Geti, + fxt_detection_to_classification_project: Project, + mocker: MockerFixture + ): + # Arrange + mock_get_project_by_name = mocker.patch( + "geti_sdk.geti.ProjectClient.get_project_by_name", + return_value=fxt_detection_to_classification_project, + ) + fetched_images = (mocker.MagicMock(),) + mock_image_client_get_all = mocker.patch( + "geti_sdk.geti.ImageClient._get_all", + return_value=fetched_images, + ) + model_client_object_mock = mocker.MagicMock() + mocked_model_client = mocker.patch( + "geti_sdk.benchmarking.benchmarker.ModelClient", return_value=model_client_object_mock + ) + active_models = (mocker.MagicMock(), mocker.MagicMock()) + model_client_object_mock.get_all_active_models.return_value = active_models + + mocked_training_client = mocker.patch( + "geti_sdk.benchmarking.benchmarker.TrainingClient" + ) + project_name = "project name" + precision_levels = ["PRECISION_1", "PRECISION_2"] + + # Act + # Multi task project, no media provided + benchmarker = Benchmarker( + geti=fxt_mocked_geti, + project=project_name, + precision_levels=precision_levels, + ) + + # Assert + mock_get_project_by_name.assert_called_once_with(project_name=project_name) + mock_image_client_get_all.assert_called_once() + mocked_model_client.assert_called_once() + model_client_object_mock.get_all_active_models.assert_called_once() + model_client_object_mock.get_task_for_model.assert_called_with(model=active_models[1]) + mocked_training_client.assert_called_once() + assert not benchmarker._is_single_task + assert benchmarker._task_chain_algorithms == [[active_models[0].architecture], [active_models[1].architecture]] + assert benchmarker._task_chain_models == [[active_models[0]], [active_models[1]]] + assert tuple(benchmarker.images) == fetched_images + assert benchmarker._are_models_specified + + def test_set_task_chain_models(self, fxt_benchmarker_task_chain: Benchmarker, mocker: MockerFixture): + # Arrange + models_task_1 = (mocker.MagicMock(), mocker.MagicMock()) + models_task_2 = (mocker.MagicMock(), mocker.MagicMock()) + + # Act + fxt_benchmarker_task_chain.set_task_chain_models(models_task_1, models_task_2) + + # Assert + # raise Exception(fxt_benchmarker_task_chain._task_chain_models) + assert fxt_benchmarker_task_chain._are_models_specified + assert fxt_benchmarker_task_chain._task_chain_models == [ + [models_task_1[0], models_task_2[0]], + [models_task_1[0], models_task_2[1]], + [models_task_1[1], models_task_2[0]], + [models_task_1[1], models_task_2[1]], + ] + assert fxt_benchmarker_task_chain._task_chain_algorithms == [ + [models_task_1[0].architecture, models_task_2[0].architecture], + [models_task_1[0].architecture, models_task_2[1].architecture], + [models_task_1[1].architecture, models_task_2[0].architecture], + [models_task_1[1].architecture, models_task_2[1].architecture], + ] + + def test_set_task_chain_algorithms(self, fxt_benchmarker_task_chain: Benchmarker, mocker: MockerFixture): + # Arrange + algorithms_task_1 = ("ALGO_1", "ALGO_2") + algorithms_task_2 = ("ALGO_3", "ALGO_4") + + # Act + fxt_benchmarker_task_chain.set_task_chain_algorithms(algorithms_task_1, algorithms_task_2) + + # Assert + assert not fxt_benchmarker_task_chain._are_models_specified + assert fxt_benchmarker_task_chain._task_chain_models is None + assert fxt_benchmarker_task_chain._task_chain_algorithms == [ + ["ALGO_1", "ALGO_3"], + ["ALGO_1", "ALGO_4"], + ["ALGO_2", "ALGO_3"], + ["ALGO_2", "ALGO_4"], + ] + + def test_properties(self, fxt_benchmarker_task_chain: Benchmarker): + # Assert + assert fxt_benchmarker_task_chain.models == fxt_benchmarker_task_chain._task_chain_models + assert fxt_benchmarker_task_chain.algorithms == fxt_benchmarker_task_chain._task_chain_algorithms + with pytest.raises(ValueError): + # benchmarker has not been initialized + fxt_benchmarker_task_chain.optimized_models + + def test__train_model_for_algorithm(self, fxt_benchmarker_task_chain: Benchmarker): + # Arrange + task_index = 0 + algorithm_name = 'ALGO_1' + + # Act + model = fxt_benchmarker_task_chain._train_model_for_algorithm(task_index, algorithm_name) + + # Assert + fxt_benchmarker_task_chain.training_client.train_task.assert_called_once() + assert model is fxt_benchmarker_task_chain.model_client.get_model_for_job() + + def test__optimize_model_for_algorithm(self, fxt_benchmarker_task_chain: Benchmarker, mocker: MockerFixture): + # Arrange + model = mocker.MagicMock() + precision = "INT8" + + # Act + optimized_model = fxt_benchmarker_task_chain._optimize_model_for_algorithm(model, precision) + + # Assert + fxt_benchmarker_task_chain.model_client.optimize_model.assert_called_once_with(model=model, optimization_type="pot") + assert optimized_model is fxt_benchmarker_task_chain.model_client.update_model_detail().get_optimized_model() + + def test_prepare_benchmark(self, fxt_benchmarker: Benchmarker, mocker: MockerFixture, fxt_temp_directory: str): + # Arrange + mock_deploy_project = mocker.patch.object(fxt_benchmarker.geti, "deploy_project") + + # Act + fxt_benchmarker.prepare_benchmark(fxt_temp_directory) + + # Assert + assert mock_deploy_project.call_count == len(fxt_benchmarker.models) * len(fxt_benchmarker.precision_levels) + assert fxt_benchmarker._optimized_models is not None + assert fxt_benchmarker._deployment_folders == [ + str(Path(fxt_temp_directory) / f"deployment_{i}") + for i in range(len(fxt_benchmarker._optimized_models)) + ] + + def test_initialize_from_folder(self, fxt_benchmarker: Benchmarker, mocker: MockerFixture, fxt_temp_directory: str): + # Arrange + deployment_path = Path(fxt_temp_directory) / "deployment_test" + deployment_path.mkdir() + deployment = mocker.MagicMock() + deployment.project.name = fxt_benchmarker.project.name + mocked_deployment_from_folder = mocker.patch( + "geti_sdk.benchmarking.benchmarker.Deployment.from_folder", + return_value=deployment + ) + + # Act + fxt_benchmarker.initialize_from_folder(fxt_temp_directory) + + # Assert + mocked_deployment_from_folder.assert_called_once_with(path_to_folder=str(deployment_path)) + assert len(fxt_benchmarker._deployment_folders) == 1 + assert fxt_benchmarker._deployment_folders[0] == str(deployment_path) + + def test_throughput_benchmark(self, fxt_benchmarker: Benchmarker, mocker: MockerFixture, fxt_temp_directory: str): + # Arrange + _ = mocker.patch.object(fxt_benchmarker.geti, "deploy_project") + loaded_images = [mocker.MagicMock(), mocker.MagicMock()] + mock_load_benchmark_media = mocker.patch( + "geti_sdk.benchmarking.benchmarker.load_benchmark_media", + return_value=loaded_images + ) + fxt_benchmarker.prepare_benchmark(fxt_temp_directory) + deployment = mocker.MagicMock() + deployment.project.name = fxt_benchmarker.project.name + deployment.models = [mocker.MagicMock(),] + _ = mocker.patch( + "geti_sdk.benchmarking.benchmarker.Deployment.from_folder", + return_value=deployment + ) + + number_of_runs = len(fxt_benchmarker.models) * len(fxt_benchmarker.precision_levels) + frames, repeats = len(loaded_images), 2 + results_file = Path(fxt_temp_directory) / "results.csv" + + # Act + fxt_benchmarker.run_throughput_benchmark( + working_directory=fxt_temp_directory, results_filename=results_file.stem, + frames=frames, repeats=repeats + ) + + # Assert + mock_load_benchmark_media.assert_called_once() + assert results_file.is_file() + assert len(list(csv.DictReader(results_file.open()))) == number_of_runs + assert deployment.load_inference_models.call_count == number_of_runs + # For each model infer is called: 1 Warm-up call, 1 time estimation call and `frames * repeats` for benchmark + assert deployment.infer.call_count == number_of_runs * (2 + frames * repeats)