From fc9cdb42ef982ad76e063e1895305f40a953c4d6 Mon Sep 17 00:00:00 2001 From: GitHub Actions Stats Bot <> Date: Mon, 3 Feb 2025 02:16:31 +0000 Subject: [PATCH 01/10] automatic update of stats files --- data/stats_current_test_info.yml | 88 +++++++++++------------------ data/stats_weekly_data.yml | 95 ++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 56 deletions(-) diff --git a/data/stats_current_test_info.yml b/data/stats_current_test_info.yml index 5d34b23667..142ef5beeb 100644 --- a/data/stats_current_test_info.yml +++ b/data/stats_current_test_info.yml @@ -1,7 +1,7 @@ summary: - content_total: 323 + content_total: 324 content_with_all_tests_passing: 31 - content_with_tests_enabled: 33 + content_with_tests_enabled: 57 sw_categories: automotive: {} cross-platform: @@ -31,28 +31,23 @@ sw_categories: - fedora:latest: passed ambaviz: readable_title: Arm AMBA Viz - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] ams: readable_title: Arm Performance Studio - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] anaconda: readable_title: Anaconda tests_and_status: - - ubuntu:latest: failed + - ubuntu:latest: passed ansible: readable_title: Ansible - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] aperf: readable_title: AWS Perf (APerf) - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] arduino-pico: readable_title: Arduino core for the Raspberry Pi Pico - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] arm-gnu: readable_title: Arm GNU Toolchain tests_and_status: @@ -60,36 +55,29 @@ sw_categories: - fedora:latest: passed armclang: readable_title: Arm Compiler for Embedded - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] armds: readable_title: Arm Development Studio - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] armie: - readable_title: Arm Instruction Emulator - tests_and_status: - - ubuntu:latest: passed + readable_title: Arm Instruction Emulator (armie) + tests_and_status: [] armpl: readable_title: Arm Performance Libraries - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] aws-cli: readable_title: AWS CLI - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] aws-copilot: readable_title: AWS Copilot CLI - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] azure-cli: readable_title: Azure CLI tests_and_status: - ubuntu:latest: passed bolt: readable_title: BOLT - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] cross: readable_title: Cross-compiler tests_and_status: @@ -109,8 +97,7 @@ sw_categories: - ubuntu:latest: passed gcloud: readable_title: Google Cloud Platform (GCP) CLI - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] gfortran: readable_title: GFortran tests_and_status: @@ -124,8 +111,7 @@ sw_categories: - ubuntu:latest: passed multipass: readable_title: Multipass - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] native: readable_title: Native compiler tests_and_status: @@ -137,55 +123,45 @@ sw_categories: - ubuntu:latest: passed papi: readable_title: Performance API (PAPI) - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] perf: readable_title: Perf for Linux on Arm (LinuxPerf) - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] pulumi: readable_title: Pulumi - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] pytorch: readable_title: PyTorch tests_and_status: - ubuntu:latest: passed rust: readable_title: Rust for Linux Applications - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] rust_embedded: readable_title: Rust for Embedded Applications - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] ssh: readable_title: SSH - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] streamline-cli: readable_title: Streamline CLI Tools - tests_and_status: - - ubuntu:latest: passed - sysbox: - readable_title: Sysbox - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] swift: readable_title: Swift tests_and_status: [] + sysbox: + readable_title: Sysbox + tests_and_status: [] terraform: readable_title: Terraform tests_and_status: - ubuntu:latest: passed - vnc: - readable_title: VNC on Arm Linux - tests_and_status: - - ubuntu:latest: passed topdown-tool: readable_title: Telemetry Solution (Topdown Methodology) - tests_and_status: - - ubuntu:latest: passed + tests_and_status: [] + vnc: + readable_title: VNC on Arm Linux + tests_and_status: [] iot: {} laptops-and-desktops: {} mobile-graphics-and-gaming: {} diff --git a/data/stats_weekly_data.yml b/data/stats_weekly_data.yml index d4b8ae9b85..b0c9d79f82 100644 --- a/data/stats_weekly_data.yml +++ b/data/stats_weekly_data.yml @@ -4811,3 +4811,98 @@ avg_close_time_hrs: 0 num_issues: 15 percent_closed_vs_total: 0.0 +- a_date: '2025-02-03' + content: + automotive: 1 + cross-platform: 28 + embedded-and-microcontrollers: 40 + install-guides: 93 + iot: 5 + laptops-and-desktops: 34 + mobile-graphics-and-gaming: 26 + servers-and-cloud-computing: 97 + total: 324 + contributions: + external: 45 + internal: 373 + github_engagement: + num_forks: 30 + num_prs: 9 + individual_authors: + alaaeddine-chakroun: 2 + alexandros-lamprineas: 1 + annie-tallund: 1 + arm: 3 + arnaud-de-grandmaison: 1 + arnaud-de-grandmaison,-paul-howard,-and-pareena-verma: 1 + basma-el-gaabouri: 1 + ben-clark: 1 + bolt-liu: 2 + brenda-strech: 1 + chaodong-gong,-alex-su,-kieran-hejmadi: 1 + chen-zhang: 1 + christopher-seidl: 7 + cyril-rohr: 1 + daniel-gubay: 1 + daniel-nguyen: 1 + david-spickett: 2 + dawid-borycki: 31 + diego-russo: 1 + diego-russo-and-leandro-nunes: 1 + elham-harirpoush: 2 + florent-lebeau: 5 + "fr\xE9d\xE9ric--lefred--descamps": 2 + gabriel-peterson: 5 + gayathri-narayana-yegna-narayanan: 1 + georgios-mermigkis-and-konstantinos-margaritis,-vectorcamp: 1 + graham-woodward: 1 + han-yin: 1 + iago-calvo-lista,-arm: 1 + james-whitaker,-arm: 1 + jason-andrews: 95 + joe-stech: 1 + johanna-skinnider: 2 + jonathan-davies: 2 + jose-emilio-munoz-lopez,-arm: 1 + julie-gaskin: 4 + julio-suarez: 5 + kasper-mecklenburg: 1 + kieran-hejmadi: 1 + koki-mitsunami: 2 + konstantinos-margaritis: 7 + kristof-beyls: 1 + liliya-wu: 1 + mathias-brossard: 1 + michael-hall: 5 + nikhil-gupta,-pareena-verma,-nobel-chowdary-mandepudi,-ravi-malhotra: 1 + nobel-chowdary-mandepudi: 1 + odin-shen: 2 + owen-wu,-arm: 2 + pareena-verma: 34 + pareena-verma,-annie-tallund: 1 + pareena-verma,-jason-andrews,-and-zach-lasiuk: 1 + pareena-verma,-joe-stech,-adnan-alsinan: 1 + paul-howard: 1 + pranay-bakre: 4 + pranay-bakre,-masoud-koleini,-nobel-chowdary-mandepudi,-na-li: 1 + preema-merlin-dsouza: 1 + przemyslaw-wirkus: 2 + rin-dobrescu: 1 + roberto-lopez-mendez: 2 + ronan-synnott: 46 + thirdai: 1 + tianyu-li: 1 + tom-pilar,-daniel-nguyen: 1 + uma-ramalingam: 1 + varun-chari,-albin-bernhardsson: 1 + varun-chari,-pareena-verma: 1 + visualsilicon: 1 + willen-yang: 1 + ying-yu: 1 + ying-yu,-arm: 1 + zach-lasiuk: 1 + zhengjun-xing: 2 + issues: + avg_close_time_hrs: 0 + num_issues: 12 + percent_closed_vs_total: 0.0 From 0af2955406ccf06eb4979b0c74da83459a89d237 Mon Sep 17 00:00:00 2001 From: Maddy Underwood Date: Fri, 7 Feb 2025 18:18:20 +0000 Subject: [PATCH 02/10] First pass review. --- .../android_opencv_kleidicv/_index.md | 2 +- .../android_opencv_kleidicv/background.md | 17 ++++++++++------- .../android_opencv_kleidicv/create-project.md | 18 +++++++++--------- .../android_opencv_kleidicv/process-images.md | 16 +++++++++------- .../android_opencv_kleidicv/summary.md | 4 ++-- .../android_opencv_kleidicv/ui.md | 11 ++++++----- 6 files changed, 37 insertions(+), 31 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md index 9894190fbb..c54a28477b 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md @@ -3,7 +3,7 @@ title: Accelerate OpenCV-based Android Applications with KleidiCV minutes_to_complete: 45 -who_is_this_for: This is an introductory topic for developers who are interested in creating Computer Vision Applications with OpenCV and KleidiCV on Android Devices. +who_is_this_for: This is an introductory topic for developers who are interested in creating Computer Vision applications with OpenCV and KleidiCV on Android Devices. learning_objectives: - Describe what KleidiCV is, and what it can offer. diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md index 3688888ec9..3c2142da79 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md @@ -10,22 +10,25 @@ layout: "learningpathall" ## OpenCV Overview with HAL and KleidiCV ### OpenCV in Computer Vision -OpenCV (Open Source Computer Vision Library) is a robust framework for real-time computer vision, widely used across platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, making OpenCV a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work seamlessly across multiple devices without extensive code modifications. +OpenCV (Open Source Computer Vision Library) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, making OpenCV a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. ### Integration with Android and Kotlin -OpenCV integrates well with Android development, leveraging Kotlin’s concise syntax and Java interoperability to simplify implementation. Developers can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. +OpenCV integrates with Android development, leveraging Kotlin’s concise syntax and Java's interoperability to simplify implementation. You can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. ### Performance Optimization and HAL -OpenCV is optimized for high performance, utilizing native C++ code for efficient processing. This ensures real-time performance, particularly on mobile devices where computational resources may be limited. A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, reducing processing time and power consumption. HAL makes OpenCV highly adaptable to modern multi-core processors and GPU architectures, vital for computationally intensive tasks like object detection and image recognition. +OpenCV is optimized for performance, utilizing native C++ code for efficient processing. This ensures real-time performance, particularly on mobile devices where computational resources might be limited. -### ARM and KleidiCV -To further enhance OpenCV’s capabilities on ARM-based devices, ARM developed KleidiCV, a specialized library that leverages OpenCV’s HAL for hardware acceleration. KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines. KleidiCV focuses on optimizing various OpenCV functions specifically for ARM processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of ARM hardware, making it an ideal solution for applications requiring high-performance computer vision on ARM-based devices. +A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, reducing processing time and power consumption. HAL makes OpenCV highly adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks such as object detection and image recognition. + +### Arm and KleidiCV + +To further enhance OpenCV’s capabilities on Arm-based devices, Arm developed KleidiCV, a specialized library that leverages OpenCV’s HAL for hardware acceleration. KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. ### OpenCV for Android Developers For Android developers, OpenCV offers an SDK designed for seamless integration with Android Studio, the primary development environment. The SDK simplifies the process of adding OpenCV libraries, managing dependencies, and configuring projects, enabling developers to incorporate sophisticated vision capabilities into their applications effortlessly. -By combining OpenCV’s rich feature set, hardware optimization via HAL, and specialized enhancements like ARM’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. +By combining OpenCV’s rich feature set, hardware optimization from HAL, and specialized enhancements such as Arm’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. -In this Learning Path we will demonstrate how to use KleidiCV-accelerated OpenCV in Android application. +In this Learning Path, you will see how you can use KleidiCV-accelerated OpenCV in an Android application. You can find all the code used in this Learning Path in a [GitHub repository](https://github.com/dawidborycki/Arm64.KleidiCV.Demo.git). diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md index fbcd527a9b..77f559659d 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md @@ -7,16 +7,16 @@ weight: 3 layout: "learningpathall" --- ## Create a project -You will need a development computer with [Android Studio](https://developer.android.com/studio) installed (this examples uses Android Studio Ladybug | 2024.2.1 Patch 3) +You will need a development computer with [Android Studio](https://developer.android.com/studio) installed. This examples uses Android Studio Ladybug 2024.2.1, Patch 3. Follow these steps to create a project and add OpenCV with KleidiCV support: -1. Open Android Studio on your development machine and then click the **+ New Project** icon: -2. In the New Project window, select **Empty Views Activity**: +1. Open Android Studio on your development machine, and then click the **+ New Project** icon. +2. In the **New Project** window, select **Empty Views Activity**: -![img1](Figures/01.png) +![img1 alt-text#center](Figures/01.png "Figure 1. Creating a new project.") -3. Configure the project as follows (see figure below): +3. Figure 2 shows you how to configure the project: - Name: **Arm64.KleidiCV.Demo**. - Package name: **com.arm.arm64kleidicvdemo**. - Save location: *Select relevant file location*. @@ -24,11 +24,11 @@ Follow these steps to create a project and add OpenCV with KleidiCV support: - Minimum SDK: **API 24**. - Build configuration language: **Kotlin DSL**. -![img2](Figures/02.png) +![img2 alt-text#center](Figures/02.png "Figure 2. Configuring your new project.") 4. Click the **Finish** button. -The project will be ready in a few moments. Afterward you can configure the project. +It takes a few moments and then the project is ready. You will be able to configure the project further later on. ## Add OpenCV support To add OpenCV for Arm64, open the *build.gradle.ts (Module: app)*, and add the following line under the dependencies: @@ -37,7 +37,7 @@ To add OpenCV for Arm64, open the *build.gradle.ts (Module: app)*, and add the f implementation("org.opencv:opencv:4.11.0") ``` -Also, make sure that compileSdk is set to 35. The contents of the file should look something like this: +Make sure that compileSdk is set to 35. The contents of the file should look like this: ```JSON plugins { @@ -90,6 +90,6 @@ dependencies { } ``` -Then, click the **Sync Now** link in the top pane that appears. From here on, you can use OpenCV with KleidiCV support in your application. +Now click the **Sync Now** link in the top pane that appears. From here on, you can use OpenCV with KleidiCV support in your application. Save the file. In the next step, you will define the application UI. \ No newline at end of file diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index be233058ce..48be2c2599 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -10,9 +10,9 @@ layout: "learningpathall" ## Process images In this final step, you will implement the application logic to process the images. -Start by adding a `assets` folder under `src/main`. Then, under `assets` folder add a `img.png` image. This image can be any kind of image as it converted to the right type by the app. We will use this image for processing. Here, we use the [cameraman image](https://github.com/antimatter15/cameraman). +Start by adding an `assets` folder under `src/main`. Then, under the `assets` folder add an `img.png` image. This image can be any kind of image as it converted to the right type by the app. We will use this image for processing. Here, we use the [cameraman image](https://github.com/antimatter15/cameraman). -To make navigation betwee files easier in Android Studio, select "Project" from the project browser pane. +To make navigation between files easier in Android Studio, select **Project** from the project browser pane. ## ImageOperation You will now create an enumeration (enum class) for a set of image processing operations in an application that uses the OpenCV library. Under the `src/main/java/com/arm/arm64kleidicvdemo` add the `ImageOperation.kt` file and modify it as follows: @@ -62,7 +62,7 @@ enum class ImageOperation(val displayName: String) { } ``` -The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName (a user-friendly string describing the operation) and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is will be available in the dst parameter of the apply method. +The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation) and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is will be available in the dst parameter of the apply method. Here we have four constants: 1. GAUSSIAN_BLUR. Applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. @@ -387,24 +387,26 @@ You can launch the application in emulator or on the actual device. When you do ![img6](Figures/06.jpg) ## Performance uplift -To appreciate the performance uplift oferred by KleidiCV you can now switch to one of the earliest OpenCV versions, which do not have KleidiCV, e.g. 4.9.0, and re-run the application. To do so open the build.gradle.kts, and modify this line +To appreciate the performance uplift offered by KleidiCV, now switch to one of the earliest OpenCV versions, which does not have KleidiCV. Version 4.9.0 is an example of one to try. + +To rerun the application open build.gradle.kts, and modify this line, from: ```XML implementation("org.opencv:opencv:4.11.0") ``` -to +to: ```XML implementation("org.opencv:opencv:4.9.0") ``` -Then, click the Sync Now button, and deploy the app to the Android device (here we used Samsung Galaxy S22): +Now click the **Sync Now** button, and deploy the app to the Android device. This example used a Samsung Galaxy S22: ![img7](Figures/07.jpg) ![img8](Figures/08.jpg) ![img9](Figures/09.jpg) ![img10](Figures/10.jpg) -We achieved the following performance uplift: +This achieved the following performance uplift: | Operation | Average computation time [ms] | Performance Uplift | |-----------|------------------|------------------|---------------------| diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md index b5f7f3df5b..98ceff5da0 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md @@ -8,6 +8,6 @@ layout: "learningpathall" --- ## Summary ## -In this Learning Path, you have created an Android application to processimages using OpenCV accelerated with KleidiCV. +In this Learning Path, you have created an Android application to process images using OpenCV accelerated with KleidiCV. -The application you created demonstrates image processing using OpenCV, focusing on efficiency and modularity. The application includes an ImageOperation enum to define various processing tasks, such as Gaussian blur, resizing, and rotation, each encapsulated with its implementation logic. The ImageProcessor class applies these operations on OpenCV Mat objects, while the PerformanceMetrics class calculates and displays statistical metrics like average and standard deviation of operation durations. The MainActivity class orchestrates the user interface and interaction, allowing users to load an image, select a processing operation from a dropdown menu, and view the processed result along with performance metrics. The design is clean and extensible, highlighting OpenCV’s capabilities and measuring their performance on ARM-based devices. +The application you created demonstrates image processing using OpenCV, focusing on efficiency and modularity. The application includes an ImageOperation enum to define various processing tasks, such as Gaussian blur, resizing, and rotation, each encapsulated with its implementation logic. The ImageProcessor class applies these operations on OpenCV Mat objects, while the PerformanceMetrics class calculates and displays statistical metrics like average and standard deviation of operation durations. The MainActivity class orchestrates the user interface and interaction, allowing users to load an image, select a processing operation from a drop-down menu, and view the processed result along with performance metrics. The design is clean and extensible, highlighting OpenCV’s capabilities and measuring their performance on Arm-based devices. diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md index 4c5966eaf0..46dc1da2bc 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md @@ -6,10 +6,11 @@ weight: 4 layout: "learningpathall" --- +## Modify the application view + You will now define the application UI. -## Modify the application view -To modify the application view, open `app/src/main/res/layout/activity_main.xmlactivity_main.xml` and replace the file contents with the following code: +To modify the application view, open `app/src/main/res/layout/activity_main.xml` and replace the file contents with the following code: ```XML @@ -86,10 +87,10 @@ To modify the application view, open `app/src/main/res/layout/activity_main.xmla This XML layout defines the user interface (UI) for an Android application. The root layout is a ConstraintLayout. Within the ConstraintLayout, a LinearLayout is embedded as the primary container for the UI components. This LinearLayout is oriented vertically, ensuring that its child elements are stacked one below the other. Inside the LinearLayout, the following components are defined: -1. ImageView for Displaying Images. We will use it to display an original and processed image. -2. Load Image Button. We use this button to load an image. +1. ImageView for Displaying Images. You will use this component to display an original and processed image. +2. Load Image Button. This button is for loading an image. 3. Spinner for Selecting Image Processing Options. -4. Process Button. Another Button, which will trigger the image processing. +4. Process Button. You can use this button for triggering image processing. 5. TextView to display computation time. In the next step, you will implement the application logic. \ No newline at end of file From 82bff7b0cf535b39567b107b3238a2ca7d7e8eb6 Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sat, 8 Feb 2025 07:05:38 +0000 Subject: [PATCH 03/10] Added prereq about Android dev concepts. Changed OS to Android. Tweaking language. --- .../android_opencv_kleidicv/_index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md index c54a28477b..a37deced22 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md @@ -12,6 +12,7 @@ learning_objectives: prerequisites: - A development machine with [Android Studio](https://developer.android.com/studio) installed. + - Familiarity with Android development concepts. - An Android smartphone. author: Dawid Borycki @@ -22,7 +23,7 @@ subjects: Graphics armips: - Cortex-A operatingsystems: - - Windows + - Android tools_software_languages: - Android - Android Studio From 16703184bf01b38f4ba635df056449e7559496d7 Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sat, 8 Feb 2025 07:05:52 +0000 Subject: [PATCH 04/10] Further improvements. --- .../android_opencv_kleidicv/background.md | 11 +++++----- .../android_opencv_kleidicv/create-project.md | 18 +++++++++------ .../android_opencv_kleidicv/process-images.md | 22 ++++++++++--------- .../android_opencv_kleidicv/summary.md | 2 +- .../android_opencv_kleidicv/ui.md | 2 +- 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md index 3c2142da79..5d5cee8ac4 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md @@ -10,7 +10,7 @@ layout: "learningpathall" ## OpenCV Overview with HAL and KleidiCV ### OpenCV in Computer Vision -OpenCV (Open Source Computer Vision Library) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, making OpenCV a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. +Open Source Computer Vision Library (OpenCV)) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, making OpenCV a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. ### Integration with Android and Kotlin OpenCV integrates with Android development, leveraging Kotlin’s concise syntax and Java's interoperability to simplify implementation. You can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. @@ -18,17 +18,16 @@ OpenCV integrates with Android development, leveraging Kotlin’s concise syntax ### Performance Optimization and HAL OpenCV is optimized for performance, utilizing native C++ code for efficient processing. This ensures real-time performance, particularly on mobile devices where computational resources might be limited. -A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, reducing processing time and power consumption. HAL makes OpenCV highly adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks such as object detection and image recognition. +A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, reducing processing time and power consumption. HAL makes OpenCV adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks such as object detection and image recognition. ### Arm and KleidiCV - -To further enhance OpenCV’s capabilities on Arm-based devices, Arm developed KleidiCV, a specialized library that leverages OpenCV’s HAL for hardware acceleration. KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. +Arm developed KleidiCV, a specialized library that leverages OpenCV’s HAL for hardware acceleration. KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. ### OpenCV for Android Developers For Android developers, OpenCV offers an SDK designed for seamless integration with Android Studio, the primary development environment. The SDK simplifies the process of adding OpenCV libraries, managing dependencies, and configuring projects, enabling developers to incorporate sophisticated vision capabilities into their applications effortlessly. -By combining OpenCV’s rich feature set, hardware optimization from HAL, and specialized enhancements such as Arm’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. +By combining OpenCV’s feature set, hardware optimization from HAL, and specialized enhancements such as Arm’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. -In this Learning Path, you will see how you can use KleidiCV-accelerated OpenCV in an Android application. +In this Learning Path, you will discover how you can use KleidiCV-accelerated OpenCV in an Android application. You can find all the code used in this Learning Path in a [GitHub repository](https://github.com/dawidborycki/Arm64.KleidiCV.Demo.git). diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md index 77f559659d..d6f264579a 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md @@ -7,11 +7,13 @@ weight: 3 layout: "learningpathall" --- ## Create a project -You will need a development computer with [Android Studio](https://developer.android.com/studio) installed. This examples uses Android Studio Ladybug 2024.2.1, Patch 3. +You will need a development computer with [Android Studio](https://developer.android.com/studio) installed. This example uses Android Studio Ladybug 2024.2.1, Patch 3. -Follow these steps to create a project and add OpenCV with KleidiCV support: +You can create a project and add OpenCV with KleidiCV support by following the steps outlined in this section. -1. Open Android Studio on your development machine, and then click the **+ New Project** icon. +Start by creating a new project, and configuring it: + +1. Open Android Studio on your development machine, and click the **+ New Project** icon. 2. In the **New Project** window, select **Empty Views Activity**: ![img1 alt-text#center](Figures/01.png "Figure 1. Creating a new project.") @@ -19,7 +21,7 @@ Follow these steps to create a project and add OpenCV with KleidiCV support: 3. Figure 2 shows you how to configure the project: - Name: **Arm64.KleidiCV.Demo**. - Package name: **com.arm.arm64kleidicvdemo**. -- Save location: *Select relevant file location*. +- Save location: *Select relevant file location for your setup*. - Language: **Kotlin**. - Minimum SDK: **API 24**. - Build configuration language: **Kotlin DSL**. @@ -28,10 +30,10 @@ Follow these steps to create a project and add OpenCV with KleidiCV support: 4. Click the **Finish** button. -It takes a few moments and then the project is ready. You will be able to configure the project further later on. +Now wait a few moments until your project is ready. You will be able to configure the project further later on. ## Add OpenCV support -To add OpenCV for Arm64, open the *build.gradle.ts (Module: app)*, and add the following line under the dependencies: +To add OpenCV support for Arm64 (AArch64), open the *build.gradle.ts (Module: app)*, and add the following line under the dependencies: ```JSON implementation("org.opencv:opencv:4.11.0") @@ -90,6 +92,8 @@ dependencies { } ``` -Now click the **Sync Now** link in the top pane that appears. From here on, you can use OpenCV with KleidiCV support in your application. +Now click the **Sync Now** link in the top pane that appears. + +You can now use OpenCV with KleidiCV support in your application. Save the file. In the next step, you will define the application UI. \ No newline at end of file diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index 48be2c2599..fe3b7c8968 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -62,7 +62,7 @@ enum class ImageOperation(val displayName: String) { } ``` -The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation) and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is will be available in the dst parameter of the apply method. +The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation) and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is available in the dst parameter of the apply method. Here we have four constants: 1. GAUSSIAN_BLUR. Applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. @@ -137,10 +137,10 @@ The above code defines a PerformanceMetrics class that calculates and represents The PerformanceMetrics class is designed to analyze and summarize performance measurements, such as execution times or latency values. It accepts a list of durations in nanoseconds, computes key metrics like average, minimum, maximum, and standard deviation, and presents them in milliseconds. -By encapsulating the raw data (durationsNano) and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden toString method makes it easy to generate a human-readable summary for reporting or debugging purposes. Specifically, we will use this method to report the performance metrics to the user. +By encapsulating the raw data (durationsNano) and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden toString method makes it easy to generate a human-readable summary for reporting or debugging purposes. Specifically, you will use this method to report the performance metrics to the user. ## MainActivity -Finally, we modify the MainActivity.kt as follows: +Finally, now modify the MainActivity.kt as follows: ```Kotlin package com.arm.arm64kleidicvdemo @@ -295,32 +295,32 @@ class MainActivity : AppCompatActivity() { The above Kotlin code defines the main activity for our application that demonstrates image processing using OpenCV. The MainActivity class extends AppCompatActivity, serving as the entry point for the app’s user interface. It manages the lifecycle of the activity and orchestrates the image processing logic. There are several members of this class: -1. viewBinding - manages UI components via the ActivityMainBinding class, simplifying access to views in the layout. +1. viewBinding - manages UI components through the ActivityMainBinding class, simplifying access to views in the layout. 2. imageProcessor - an instance of the ImageProcessor class that applies selected image operations. 3. originalMat - a Mat object representing the original image loaded from the app’s assets. 4. currentBitmap - stores the current image as a Bitmap for display. 5. REPETITIONS - number of times each operation is performed for performance measurement. 6. TEST_IMAGE - name of the test image located in the app’s assets. -When the activity starts, it sets up the user interface, initializes OpenCV and sets up UI listeners: +When the activity starts, it sets up the user interface, initializes OpenCV, and sets up UI listeners: The activity also implements several helper methods: 1. setupSpinner - populates the spinner with the names of available ImageOperation enums. 2. showToast - displays a short toast message for user feedback. -3. loadImage - loads the test image (img.png) from the app’s assets. Then, the method converts the image into a Bitmap and stores it for display. The bitmap is also converted to an OpenCV Mat object and changed its color format to grayscale (used in OpenCV). +3. loadImage - loads the test image (img.png) from the app’s assets. Then, the method converts the image into a Bitmap and stores it for display. The bitmap is also converted to an OpenCV Mat object and changed its color format to grayscale, which is used in OpenCV. 4. displayAndStoreBitmap - updates the app’s ImageView to display the loaded image. 5. convertBitmapToMat - converts the Bitmap to a Mat using OpenCV utilities. 6. processImage - this method performs the following: • Validates if an image is loaded. • Retrieves the selected operation from the spinner. • Repeats the processing operation multiple times (REPETITIONS) to measure performance. - • Calculates statistical metrics (average, min, max, standard deviation) using PerformanceMetrics. + • Calculates statistical metrics (average, min, max, and standard deviation) using PerformanceMetrics. • Updates the UI with the processed image and metrics. 7. measureOperationTime - Measures the execution time of an operation in nanoseconds using System.nanoTime(). 8. displayProcessedImage. This method converts the processed Mat back to a Bitmap for display and updates the ImageView with the processed image. ## Databinding -Finally, modify the build.gradle.kts by adding the databinding (under build features): +Finally, modify the build.gradle.kts by adding the databinding under build features: ```JSON plugins { @@ -379,7 +379,9 @@ dependencies { ``` ## Running the application -You can launch the application in emulator or on the actual device. When you do so, click the Load image button, select the image processing operation, and then click Process. You will see the processing results and a detailed performance analysis as shown in the following figures (we used Samsung Galaxy S22): +You can launch the application in an emulator or on the actual device. + +When you do so, click the **Load image** button, select the image processing operation, and then click **Process**. You will see the processing results and a detailed performance analysis as Figures 3-6 show. NOTE: This Learning Path used a Samsung Galaxy S22. ![img3](Figures/03.jpg) ![img4](Figures/04.jpg) @@ -399,7 +401,7 @@ to: implementation("org.opencv:opencv:4.9.0") ``` -Now click the **Sync Now** button, and deploy the app to the Android device. This example used a Samsung Galaxy S22: +Now click the **Sync Now** button, and deploy the app to the Android device. ![img7](Figures/07.jpg) ![img8](Figures/08.jpg) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md index 98ceff5da0..aa4867b968 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/summary.md @@ -10,4 +10,4 @@ layout: "learningpathall" In this Learning Path, you have created an Android application to process images using OpenCV accelerated with KleidiCV. -The application you created demonstrates image processing using OpenCV, focusing on efficiency and modularity. The application includes an ImageOperation enum to define various processing tasks, such as Gaussian blur, resizing, and rotation, each encapsulated with its implementation logic. The ImageProcessor class applies these operations on OpenCV Mat objects, while the PerformanceMetrics class calculates and displays statistical metrics like average and standard deviation of operation durations. The MainActivity class orchestrates the user interface and interaction, allowing users to load an image, select a processing operation from a drop-down menu, and view the processed result along with performance metrics. The design is clean and extensible, highlighting OpenCV’s capabilities and measuring their performance on Arm-based devices. +The application you created demonstrates image processing using OpenCV, focusing on efficiency and modularity. The application includes an ImageOperation enum to define various processing tasks, such as Gaussian blur, resizing, and rotation, each encapsulated with its implementation logic. The ImageProcessor class applies these operations on OpenCV Mat objects, while the PerformanceMetrics class calculates and displays statistical metrics like average and standard deviation of operation durations. The MainActivity class orchestrates the user interface and interaction, allowing users to load an image, select a processing operation from a drop-down menu, and view the processed result along with performance metrics. The design is clean and extensible, highlighting OpenCV’s capabilities and measuring the performance of these capabilities on Arm-based devices. diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md index 46dc1da2bc..7161d6dec1 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/ui.md @@ -10,7 +10,7 @@ layout: "learningpathall" You will now define the application UI. -To modify the application view, open `app/src/main/res/layout/activity_main.xml` and replace the file contents with the following code: +To modify the application view, open `app/src/main/res/layout/activity_main.xml` and replace the file contents with the following: ```XML From fdf140ca907397414ec8f839fd9b2da79812966c Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:05:32 +0000 Subject: [PATCH 05/10] Further improvements. --- .../android_opencv_kleidicv/_index.md | 2 +- .../android_opencv_kleidicv/background.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md index a37deced22..a5e9f5cccc 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md @@ -8,7 +8,7 @@ who_is_this_for: This is an introductory topic for developers who are interested learning_objectives: - Describe what KleidiCV is, and what it can offer. - Create and configure a project to add OpenCV support. - - Process images using various OpenCV functions. + - Process images using OpenCV functionality. prerequisites: - A development machine with [Android Studio](https://developer.android.com/studio) installed. diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md index 5d5cee8ac4..aefd6ebe52 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md @@ -10,15 +10,17 @@ layout: "learningpathall" ## OpenCV Overview with HAL and KleidiCV ### OpenCV in Computer Vision -Open Source Computer Vision Library (OpenCV)) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, making OpenCV a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. +Open Source Computer Vision Library (OpenCV)) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, and OpenCV is a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. ### Integration with Android and Kotlin -OpenCV integrates with Android development, leveraging Kotlin’s concise syntax and Java's interoperability to simplify implementation. You can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. +OpenCV integrates with Android development, leveraging Kotlin’s concise syntax and Java's interoperability, to simplify implementation. + +You can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. ### Performance Optimization and HAL -OpenCV is optimized for performance, utilizing native C++ code for efficient processing. This ensures real-time performance, particularly on mobile devices where computational resources might be limited. +OpenCV is optimized for performance, utilizing native C++ code for efficient processing. This ensures optimized real-time performance, particularly on mobile devices where computational resources might be limited. -A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, reducing processing time and power consumption. HAL makes OpenCV adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks such as object detection and image recognition. +A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, which reduces processing time and power consumption. HAL makes OpenCV adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks, such as object detection and image recognition. ### Arm and KleidiCV Arm developed KleidiCV, a specialized library that leverages OpenCV’s HAL for hardware acceleration. KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. From bcc72e439abd4c10aa93fd7fc6809e7d3a88f86a Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sun, 9 Feb 2025 04:06:12 +0000 Subject: [PATCH 06/10] Further updates. --- .../android_opencv_kleidicv/_index.md | 2 +- .../android_opencv_kleidicv/background.md | 29 +++++++++---------- .../android_opencv_kleidicv/create-project.md | 4 +-- .../android_opencv_kleidicv/process-images.md | 17 ++++++----- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md index a5e9f5cccc..a4a1671239 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/_index.md @@ -1,5 +1,5 @@ --- -title: Accelerate OpenCV-based Android Applications with KleidiCV +title: Accelerate an OpenCV-based Android Application with KleidiCV minutes_to_complete: 45 diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md index aefd6ebe52..2608e221f2 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md @@ -7,29 +7,28 @@ weight: 2 layout: "learningpathall" --- -## OpenCV Overview with HAL and KleidiCV +## What is OpenCV in Computer Vision? +Open Source Computer Vision Library (OpenCV) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, and OpenCV is a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. -### OpenCV in Computer Vision -Open Source Computer Vision Library (OpenCV)) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, and OpenCV is a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. +### What does OpenCV offer Android Developers? +For Android developers, OpenCV offers an SDK designed for integration with Android Studio, the primary development environment. The SDK simplifies the process of adding OpenCV libraries, managing dependencies, and configuring projects, enabling developers to easily incorporate vision capabilities into their applications. -### Integration with Android and Kotlin -OpenCV integrates with Android development, leveraging Kotlin’s concise syntax and Java's interoperability, to simplify implementation. +## How does OpenCV Integrate with Android and Kotlin? +OpenCV integrates with Android development, leveraging Kotlin’s concise syntax and Java's interoperability, to simplify implementation. You can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. -You can use OpenCV’s diverse set of functionalities, such as image and video capture, filtering, transformations, feature detection, object recognition, and machine learning integration, to build efficient and maintainable applications. - -### Performance Optimization and HAL +## How is Performance Optimized with HAL? OpenCV is optimized for performance, utilizing native C++ code for efficient processing. This ensures optimized real-time performance, particularly on mobile devices where computational resources might be limited. -A critical component enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, which reduces processing time and power consumption. HAL makes OpenCV adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks, such as object detection and image recognition. +A critical component in enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, which reduces processing time and power consumption. HAL makes OpenCV adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks, such as object detection and image recognition. -### Arm and KleidiCV -Arm developed KleidiCV, a specialized library that leverages OpenCV’s HAL for hardware acceleration. KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. +### What is KleidiCV? +KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines that leverages OpenCV’s HAL for hardware acceleration. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. -### OpenCV for Android Developers -For Android developers, OpenCV offers an SDK designed for seamless integration with Android Studio, the primary development environment. The SDK simplifies the process of adding OpenCV libraries, managing dependencies, and configuring projects, enabling developers to incorporate sophisticated vision capabilities into their applications effortlessly. +This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. -By combining OpenCV’s feature set, hardware optimization from HAL, and specialized enhancements such as Arm’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. +### A Performant Solution +By combining OpenCV’s feature set, the hardware optimization from HAL, and specialized enhancements such as Arm’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. -In this Learning Path, you will discover how you can use KleidiCV-accelerated OpenCV in an Android application. +In this Learning Path, you will learn how you can use KleidiCV-accelerated OpenCV in an Android application. You can find all the code used in this Learning Path in a [GitHub repository](https://github.com/dawidborycki/Arm64.KleidiCV.Demo.git). diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md index d6f264579a..9b10a48d17 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/create-project.md @@ -16,7 +16,7 @@ Start by creating a new project, and configuring it: 1. Open Android Studio on your development machine, and click the **+ New Project** icon. 2. In the **New Project** window, select **Empty Views Activity**: -![img1 alt-text#center](Figures/01.png "Figure 1. Creating a new project.") +![img1 alt-text#center](Figures/01.png "Figure 1: Creating a new project.") 3. Figure 2 shows you how to configure the project: - Name: **Arm64.KleidiCV.Demo**. @@ -26,7 +26,7 @@ Start by creating a new project, and configuring it: - Minimum SDK: **API 24**. - Build configuration language: **Kotlin DSL**. -![img2 alt-text#center](Figures/02.png "Figure 2. Configuring your new project.") +![img2 alt-text#center](Figures/02.png "Figure 2: Configuring your new project.") 4. Click the **Finish** button. diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index fe3b7c8968..f1a66cabb4 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -62,15 +62,16 @@ enum class ImageOperation(val displayName: String) { } ``` -The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation) and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is available in the dst parameter of the apply method. +The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is available in the dst parameter of the apply method. Here we have four constants: -1. GAUSSIAN_BLUR. Applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. -2. SOBEL. Applies the Sobel filter to detect edges in the image. It computes the gradient in both x and y directions with an 8-bit unsigned data type (CvType.CV_16S). -3. RESIZE. Resizes the image to half its original width and height. -4. ROTATE_90. Rotates the image 90 degrees clockwise. -Each enum constant must override the abstract method apply to define its specific image processing logic. +* GAUSSIAN_BLUR. This applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. +* SOBEL. This applies the Sobel filter to detect edges in the image. It computes the gradient in both x and y directions with an 8-bit unsigned data type (CvType.CV_16S). +* RESIZE. This resizes the image to half its original width and height. +* ROTATE_90. This rotates the image 90 degrees clockwise. + +Each enum constant must override the abstract method applied to define its specific image processing logic. We configured processing operations to align with current KleidiCV restrictions. Specifically, in-place changes are not supported, so the source and destination must be different images. In general, only single-channel images are supported (Gaussian blur is an exception). Sobel’s output type must be 16SC1; dx and dy must be either (1,0) or (0,1); and the border mode must be replicate. Gaussian blur supports a non-zero sigma, but its performance is best with sigma 0.0. Its uplift is most noticeable with a kernel size of 7×7. @@ -91,7 +92,7 @@ class ImageProcessor { } ``` -This Kotlin code defines a simple utility class, ImageProcessor, designed to apply a specified image processing operation on an OpenCV Mat object. The ImageProcessor class is a utility class that provides a method to process images. It doesn’t store any state or have additional properties, making it lightweight and focused on its single responsibility: applying operations to images. +This Kotlin code defines a simple utility class, ImageProcessor, designed to apply a specified image processing operation on an OpenCV Mat object. The ImageProcessor class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and focused on its single responsibility: applying operations to images. The ImageProcessor class acts as a simple orchestrator for image processing tasks. It delegates the actual processing logic to the ImageOperation enum, which encapsulates various image processing techniques. This separation of concerns keeps the ImageProcessor class focused and makes it easy to extend or modify operations by updating the ImageOperation enum. @@ -417,4 +418,4 @@ This achieved the following performance uplift: | Resize | 0,02 | 0,04 | 2x | | Rotate | 0,02 | 0,06 | 3x | -As shown above, we achieved 4× faster computations for Gaussian blur and the Sobel filter, 3× faster for rotation, and 2× faster for resizing. Note that his numbers are noise (large standard deviation), and measurements for another device can vary. \ No newline at end of file +As shown above, you achieved 4× faster computations for Gaussian blur and the Sobel filter, 3× faster for rotation, and 2× faster for resizing. Note that his numbers are noise (large standard deviation), and measurements for another device can vary. \ No newline at end of file From f8e4f12d7c60aab83d3733da0c52d4cdefb1f2c3 Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sun, 9 Feb 2025 11:43:39 +0000 Subject: [PATCH 07/10] Further refinements. --- .../android_opencv_kleidicv/process-images.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index f1a66cabb4..5908cc8a20 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -1,21 +1,25 @@ --- # User change -title: "Process Images" +title: "Processing the Images" weight: 5 layout: "learningpathall" --- -## Process images -In this final step, you will implement the application logic to process the images. +## Set up +In this final step, you will learn how to implement the application logic to process the images. -Start by adding an `assets` folder under `src/main`. Then, under the `assets` folder add an `img.png` image. This image can be any kind of image as it converted to the right type by the app. We will use this image for processing. Here, we use the [cameraman image](https://github.com/antimatter15/cameraman). +Start by adding an `assets` folder under `src/main`. Then, under the `assets` folder, add an `img.png` image file. This image can be any kind of image file, as the app will convert it to the correct type through the image processing stage. -To make navigation between files easier in Android Studio, select **Project** from the project browser pane. +In this example, the image file is a [cameraman image](https://github.com/antimatter15/cameraman). + +To facilitate easier navigation between files in Android Studio, use the **Project** menu option from the project browser pane. ## ImageOperation -You will now create an enumeration (enum class) for a set of image processing operations in an application that uses the OpenCV library. Under the `src/main/java/com/arm/arm64kleidicvdemo` add the `ImageOperation.kt` file and modify it as follows: +You will now create an enumeration, an enum class, for a set of image processing operations in an application that uses the OpenCV library. + +In the `src/main/java/com/arm/arm64kleidicvdemo` file directory, add the `ImageOperation.kt` file, and modify it as follows: ```Kotlin package com.arm.arm64kleidicvdemo @@ -62,7 +66,7 @@ enum class ImageOperation(val displayName: String) { } ``` -The ImageOperation enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation and a unique implementation of the apply method to perform the operation on an image (Mat). The processing result is available in the dst parameter of the apply method. +The `ImageOperation` enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation and a unique implementation of the apply method to perform the operation on an image (Mat). The processing results are available in the `dst` parameter of the apply method. Here we have four constants: From a5d5ab02593628eaed7d9e6ca3b81dd7a0479623 Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sun, 9 Feb 2025 12:30:26 +0000 Subject: [PATCH 08/10] Further improvements. --- .../android_opencv_kleidicv/process-images.md | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index 5908cc8a20..0e2a645606 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -70,19 +70,19 @@ The `ImageOperation` enum represents a collection of predefined image processing Here we have four constants: -* GAUSSIAN_BLUR. This applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. -* SOBEL. This applies the Sobel filter to detect edges in the image. It computes the gradient in both x and y directions with an 8-bit unsigned data type (CvType.CV_16S). -* RESIZE. This resizes the image to half its original width and height. -* ROTATE_90. This rotates the image 90 degrees clockwise. +* GAUSSIAN_BLUR. This constant applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. +* SOBEL. This constant applies the Sobel filter to detect edges in the image. It computes the gradient in both x and y directions with an 8-bit unsigned data type (CvType.CV_16S). +* RESIZE. This constant resizes the image to half its original width and height. +* ROTATE_90. This constant rotates the image 90 degrees clockwise. -Each enum constant must override the abstract method applied to define its specific image processing logic. +To define its specific image processing logic, each enum constant must override the abstract method applied. -We configured processing operations to align with current KleidiCV restrictions. Specifically, in-place changes are not supported, so the source and destination must be different images. In general, only single-channel images are supported (Gaussian blur is an exception). Sobel’s output type must be 16SC1; dx and dy must be either (1,0) or (0,1); and the border mode must be replicate. Gaussian blur supports a non-zero sigma, but its performance is best with sigma 0.0. Its uplift is most noticeable with a kernel size of 7×7. +You have configured processing operations to align with current KleidiCV restrictions. Specifically, in-place changes are not supported, so the source and destination must be different images. Generally, only single-channel images are supported (Gaussian blur is an exception). Sobel’s output type must be 16SC1; dx and dy must be either (1,0) or (0,1); and the border mode must be replicated. Gaussian blur supports a non-zero sigma, but its performance is best with sigma 0.0. Its uplift is most noticeable with a kernel size of 7×7. -There is also the companion object provides a utility method fromDisplayName. This function maps a string (displayName) to its corresponding enum constant by iterating through the list of all enum values and returns null if no match is found. +There is also the companion object that provides a utility method fromDisplayName. This function maps a string (displayName) to its corresponding enum constant by iterating through the list of all enum values, and returns null if no match is found. ## ImageProcessor -Similarly, add the ImageProcessor.kt: +Now add the ImageProcessor.kt: ```Kotlin package com.arm.arm64kleidicvdemo @@ -96,14 +96,14 @@ class ImageProcessor { } ``` -This Kotlin code defines a simple utility class, ImageProcessor, designed to apply a specified image processing operation on an OpenCV Mat object. The ImageProcessor class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and focused on its single responsibility: applying operations to images. +The Kotlin code defines a simple utility class, ImageProcessor, which applies a specified image processing operation on an OpenCV Mat object. The ImageProcessor class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and efficient. The ImageProcessor class acts as a simple orchestrator for image processing tasks. It delegates the actual processing logic to the ImageOperation enum, which encapsulates various image processing techniques. This separation of concerns keeps the ImageProcessor class focused and makes it easy to extend or modify operations by updating the ImageOperation enum. -This design is clean and modular, allowing developers to easily add new processing operations or reuse the ImageProcessor in different parts of an application. It aligns with object-oriented principles by promoting encapsulation and reducing the complexity of the processing logic. +This design is clean and modular, allowing developers to easily add new processing operations or reuse the ImageProcessor in different parts of an application. It aligns with object-oriented principles by promoting encapsulation and reducing processing logic complexity. ## PerformanceMetrics -Then, supplement the project by the `PerformanceMetrics.kt` file: +Now supplement the project with the `PerformanceMetrics.kt` file: ```Kotlin package com.arm.arm64kleidicvdemo @@ -138,9 +138,9 @@ data class PerformanceMetrics(private val durationsNano: List) { } ``` -The above code defines a PerformanceMetrics class that calculates and represents various statistical metrics for performance measurements, such as average, minimum, maximum, and standard deviation of durations. +This code defines a PerformanceMetrics class that calculates and represents various statistical metrics for performance measurements, such as average, minimum, maximum, and standard deviation of durations. -The PerformanceMetrics class is designed to analyze and summarize performance measurements, such as execution times or latency values. It accepts a list of durations in nanoseconds, computes key metrics like average, minimum, maximum, and standard deviation, and presents them in milliseconds. +The PerformanceMetrics class analyzes and summarizes performance measurements, such as execution times or latency values. It accepts a list of durations in nanoseconds, computes key metrics like average, minimum, maximum, and standard deviation, and presents them in milliseconds. By encapsulating the raw data (durationsNano) and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden toString method makes it easy to generate a human-readable summary for reporting or debugging purposes. Specifically, you will use this method to report the performance metrics to the user. @@ -300,12 +300,12 @@ class MainActivity : AppCompatActivity() { The above Kotlin code defines the main activity for our application that demonstrates image processing using OpenCV. The MainActivity class extends AppCompatActivity, serving as the entry point for the app’s user interface. It manages the lifecycle of the activity and orchestrates the image processing logic. There are several members of this class: -1. viewBinding - manages UI components through the ActivityMainBinding class, simplifying access to views in the layout. -2. imageProcessor - an instance of the ImageProcessor class that applies selected image operations. -3. originalMat - a Mat object representing the original image loaded from the app’s assets. -4. currentBitmap - stores the current image as a Bitmap for display. -5. REPETITIONS - number of times each operation is performed for performance measurement. -6. TEST_IMAGE - name of the test image located in the app’s assets. +* viewBinding - manages UI components through the ActivityMainBinding class, simplifying access to views in the layout. +* imageProcessor - an instance of the ImageProcessor class that applies selected image operations. +* originalMat - a Mat object representing the original image loaded from the app’s assets. +* currentBitmap - stores the current image as a Bitmap for display. +* REPETITIONS - number of times each operation is performed for performance measurement. +* TEST_IMAGE - name of the test image located in the app’s assets. When the activity starts, it sets up the user interface, initializes OpenCV, and sets up UI listeners: @@ -384,7 +384,7 @@ dependencies { ``` ## Running the application -You can launch the application in an emulator or on the actual device. +You can now launch the application in an emulator, or on the actual device. When you do so, click the **Load image** button, select the image processing operation, and then click **Process**. You will see the processing results and a detailed performance analysis as Figures 3-6 show. NOTE: This Learning Path used a Samsung Galaxy S22. @@ -413,7 +413,7 @@ Now click the **Sync Now** button, and deploy the app to the Android device. ![img9](Figures/09.jpg) ![img10](Figures/10.jpg) -This achieved the following performance uplift: +This particular example achieves the following performance uplift: | Operation | Average computation time [ms] | Performance Uplift | |-----------|------------------|------------------|---------------------| @@ -422,4 +422,4 @@ This achieved the following performance uplift: | Resize | 0,02 | 0,04 | 2x | | Rotate | 0,02 | 0,06 | 3x | -As shown above, you achieved 4× faster computations for Gaussian blur and the Sobel filter, 3× faster for rotation, and 2× faster for resizing. Note that his numbers are noise (large standard deviation), and measurements for another device can vary. \ No newline at end of file +As shown above, this has achieved a performance uplift of 4× faster computations for the Gaussian blur and the Sobel filter, 3× faster for rotation, and 2× faster for resizing. Note that these numbers are noise (large standard deviation), and measurements for another device might vary. \ No newline at end of file From a99900fcacd141cf534b57c0c45ac88231970dfa Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sun, 9 Feb 2025 14:02:38 +0000 Subject: [PATCH 09/10] Adding code markup --- .../android_opencv_kleidicv/process-images.md | 72 ++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index 0e2a645606..0c9741544c 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -10,14 +10,16 @@ layout: "learningpathall" ## Set up In this final step, you will learn how to implement the application logic to process the images. -Start by adding an `assets` folder under `src/main`. Then, under the `assets` folder, add an `img.png` image file. This image can be any kind of image file, as the app will convert it to the correct type through the image processing stage. +Start by adding an `assets` folder under `src/main`. + +Then, under the `assets` folder, add an `img.png` image file. This image can be any kind of image file, as the app will convert it to a specified image type through the image processing stage. In this example, the image file is a [cameraman image](https://github.com/antimatter15/cameraman). To facilitate easier navigation between files in Android Studio, use the **Project** menu option from the project browser pane. ## ImageOperation -You will now create an enumeration, an enum class, for a set of image processing operations in an application that uses the OpenCV library. +You will now create an enum class, which is an enumeration, for a set of image processing operations in an application that uses the OpenCV library. In the `src/main/java/com/arm/arm64kleidicvdemo` file directory, add the `ImageOperation.kt` file, and modify it as follows: @@ -66,23 +68,25 @@ enum class ImageOperation(val displayName: String) { } ``` -The `ImageOperation` enum represents a collection of predefined image processing operations. Each enum constant is associated with a displayName, which is a user-friendly string describing the operation and a unique implementation of the apply method to perform the operation on an image (Mat). The processing results are available in the `dst` parameter of the apply method. +The `ImageOperation` enum represents a collection of predefined image processing operations. Each enum constant is associated with a `displayName`, which is a user-friendly string describing the operation, and a unique implementation of the `apply` method to perform the operation on an image (Mat). + +The processing results are available in the `dst` parameter of the `apply` method. Here we have four constants: -* GAUSSIAN_BLUR. This constant applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. -* SOBEL. This constant applies the Sobel filter to detect edges in the image. It computes the gradient in both x and y directions with an 8-bit unsigned data type (CvType.CV_16S). -* RESIZE. This constant resizes the image to half its original width and height. -* ROTATE_90. This constant rotates the image 90 degrees clockwise. +* `GAUSSIAN_BLUR`. This constant applies a Gaussian blur to the image using a 7x7 kernel and a standard deviation of 0.0. +* `SOBEL`. This constant applies the Sobel filter to detect edges in the image. It computes the gradient in both x and y directions with an 8-bit unsigned data type (CvType.CV_16S). +* `RESIZE`. This constant resizes the image to half its original width and height. +* `ROTATE_90`. This constant rotates the image 90 degrees clockwise. To define its specific image processing logic, each enum constant must override the abstract method applied. You have configured processing operations to align with current KleidiCV restrictions. Specifically, in-place changes are not supported, so the source and destination must be different images. Generally, only single-channel images are supported (Gaussian blur is an exception). Sobel’s output type must be 16SC1; dx and dy must be either (1,0) or (0,1); and the border mode must be replicated. Gaussian blur supports a non-zero sigma, but its performance is best with sigma 0.0. Its uplift is most noticeable with a kernel size of 7×7. -There is also the companion object that provides a utility method fromDisplayName. This function maps a string (displayName) to its corresponding enum constant by iterating through the list of all enum values, and returns null if no match is found. +There is also the companion object that provides a utility method `fromDisplayName`. This function maps the string `displayName` to its corresponding enum constant by iterating through the list of all enum values, and returns null if no match is found. ## ImageProcessor -Now add the ImageProcessor.kt: +Now add the `ImageProcessor.kt`: ```Kotlin package com.arm.arm64kleidicvdemo @@ -96,11 +100,11 @@ class ImageProcessor { } ``` -The Kotlin code defines a simple utility class, ImageProcessor, which applies a specified image processing operation on an OpenCV Mat object. The ImageProcessor class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and efficient. +The Kotlin code defines a simple utility class, `ImageProcessor`, which applies a specified image processing operation on an OpenCV Mat object. The `ImageProcessor` class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and efficient. -The ImageProcessor class acts as a simple orchestrator for image processing tasks. It delegates the actual processing logic to the ImageOperation enum, which encapsulates various image processing techniques. This separation of concerns keeps the ImageProcessor class focused and makes it easy to extend or modify operations by updating the ImageOperation enum. +The `ImageProcessor` class acts as a simple orchestrator for image processing tasks. It delegates the actual processing logic to the `ImageOperation` enum, which encapsulates various image processing techniques. This separation of concerns keeps the `ImageProcessor` class focused and makes it easy to extend or modify operations by updating the `ImageOperation` enum. -This design is clean and modular, allowing developers to easily add new processing operations or reuse the ImageProcessor in different parts of an application. It aligns with object-oriented principles by promoting encapsulation and reducing processing logic complexity. +This design is clean and modular, allowing developers to easily add new processing operations or reuse the `ImageProcessor` in different parts of an application. It aligns with object-oriented principles by promoting encapsulation and reducing processing logic complexity. ## PerformanceMetrics Now supplement the project with the `PerformanceMetrics.kt` file: @@ -138,14 +142,14 @@ data class PerformanceMetrics(private val durationsNano: List) { } ``` -This code defines a PerformanceMetrics class that calculates and represents various statistical metrics for performance measurements, such as average, minimum, maximum, and standard deviation of durations. +This code defines a `PerformanceMetrics` class that calculates and represents various statistical metrics for performance measurements, such as the average, minimum, maximum, and standard deviation of durations. -The PerformanceMetrics class analyzes and summarizes performance measurements, such as execution times or latency values. It accepts a list of durations in nanoseconds, computes key metrics like average, minimum, maximum, and standard deviation, and presents them in milliseconds. +The `PerformanceMetrics` class analyzes and summarizes performance measurements, such as execution times or latency values. It accepts a list of durations in nanoseconds, computes key metrics like average, minimum, maximum, and standard deviation, and presents them in milliseconds. -By encapsulating the raw data (durationsNano) and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden toString method makes it easy to generate a human-readable summary for reporting or debugging purposes. Specifically, you will use this method to report the performance metrics to the user. +By encapsulating the raw data `durationsNano` and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden `toString` method makes it easy to generate a human-readable summary for reporting or debugging purposes. Specifically, you will use this method to report the performance metrics to the user. ## MainActivity -Finally, now modify the MainActivity.kt as follows: +Finally, now modify `MainActivity.kt` as follows: ```Kotlin package com.arm.arm64kleidicvdemo @@ -297,35 +301,35 @@ class MainActivity : AppCompatActivity() { } ``` -The above Kotlin code defines the main activity for our application that demonstrates image processing using OpenCV. The MainActivity class extends AppCompatActivity, serving as the entry point for the app’s user interface. It manages the lifecycle of the activity and orchestrates the image processing logic. +The above Kotlin code defines the main activity for our application that demonstrates image processing using OpenCV. The `MainActivity` class extends `AppCompatActivity`, serving as the entry point for the app’s user interface. It manages the lifecycle of the activity and orchestrates the image processing logic. There are several members of this class: -* viewBinding - manages UI components through the ActivityMainBinding class, simplifying access to views in the layout. -* imageProcessor - an instance of the ImageProcessor class that applies selected image operations. -* originalMat - a Mat object representing the original image loaded from the app’s assets. -* currentBitmap - stores the current image as a Bitmap for display. -* REPETITIONS - number of times each operation is performed for performance measurement. -* TEST_IMAGE - name of the test image located in the app’s assets. +* `viewBinding` - manages UI components through the `ActivityMainBinding` class, simplifying access to views in the layout. +* `imageProcessor` - an instance of the `ImageProcessor` class that applies selected image operations. +* `originalMat` - a Mat object representing the original image loaded from the app’s assets. +* `currentBitmap` - stores the current image as a Bitmap for display. +* `REPETITIONS` - number of times each operation is performed for performance measurement. +* `TEST_IMAGE` - name of the test image located in the app’s assets. When the activity starts, it sets up the user interface, initializes OpenCV, and sets up UI listeners: The activity also implements several helper methods: -1. setupSpinner - populates the spinner with the names of available ImageOperation enums. -2. showToast - displays a short toast message for user feedback. -3. loadImage - loads the test image (img.png) from the app’s assets. Then, the method converts the image into a Bitmap and stores it for display. The bitmap is also converted to an OpenCV Mat object and changed its color format to grayscale, which is used in OpenCV. -4. displayAndStoreBitmap - updates the app’s ImageView to display the loaded image. -5. convertBitmapToMat - converts the Bitmap to a Mat using OpenCV utilities. -6. processImage - this method performs the following: +1. `setupSpinner` - populates the spinner with the names of available ImageOperation enums. +2. `showToast` - displays a short toast message for user feedback. +3. `loadImage` - loads the test image (img.png) from the app’s assets. Then, the method converts the image into a Bitmap and stores it for display. The bitmap is also converted to an OpenCV Mat object and changed its color format to grayscale, which is used in OpenCV. +4. `displayAndStoreBitmap` - updates the app’s ImageView to display the loaded image. +5. `convertBitmapToMat` - converts the Bitmap to a Mat using OpenCV utilities. +6. `processImage` - this method performs the following: • Validates if an image is loaded. • Retrieves the selected operation from the spinner. - • Repeats the processing operation multiple times (REPETITIONS) to measure performance. - • Calculates statistical metrics (average, min, max, and standard deviation) using PerformanceMetrics. + • Repeats the processing operation multiple times `REPETITIONS` to measure performance. + • Calculates statistical metrics (average, min, max, and standard deviation) using `PerformanceMetrics`. • Updates the UI with the processed image and metrics. -7. measureOperationTime - Measures the execution time of an operation in nanoseconds using System.nanoTime(). -8. displayProcessedImage. This method converts the processed Mat back to a Bitmap for display and updates the ImageView with the processed image. +7. `measureOperationTime` - Measures the execution time of an operation in nanoseconds using System.nanoTime(). +8. `displayProcessedImage`. This method converts the processed Mat back to a Bitmap for display and updates the ImageView with the processed image. ## Databinding -Finally, modify the build.gradle.kts by adding the databinding under build features: +Finally, modify `build.gradle.kts` by adding the databinding under build features: ```JSON plugins { From 1555ca3c0f9ff0d3253c93687bebf7c854774fc0 Mon Sep 17 00:00:00 2001 From: Maddy Underwood <167196745+madeline-underwood@users.noreply.github.com> Date: Sun, 9 Feb 2025 16:13:25 +0000 Subject: [PATCH 10/10] Further tweaks. --- .../android_opencv_kleidicv/background.md | 6 +-- .../android_opencv_kleidicv/process-images.md | 52 +++++++++++-------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md index 2608e221f2..48fa516695 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/background.md @@ -10,7 +10,7 @@ layout: "learningpathall" ## What is OpenCV in Computer Vision? Open Source Computer Vision Library (OpenCV) is a framework for real-time computer vision, that you can use across different platforms, including mobile devices. Modern smartphones equipped with advanced cameras and powerful processors can efficiently handle complex computer vision tasks, and OpenCV is a go-to choice for developers. Its cross-platform compatibility allows the creation of versatile applications that work across multiple devices without extensive code modifications. -### What does OpenCV offer Android Developers? +## What does OpenCV offer Android Developers? For Android developers, OpenCV offers an SDK designed for integration with Android Studio, the primary development environment. The SDK simplifies the process of adding OpenCV libraries, managing dependencies, and configuring projects, enabling developers to easily incorporate vision capabilities into their applications. ## How does OpenCV Integrate with Android and Kotlin? @@ -21,12 +21,12 @@ OpenCV is optimized for performance, utilizing native C++ code for efficient pro A critical component in enabling these optimizations is the Hardware Acceleration Layer (HAL). HAL serves as an abstraction layer that enables hardware-specific acceleration by utilizing device-specific optimizations. This significantly boosts the performance of OpenCV functions on supported hardware, which reduces processing time and power consumption. HAL makes OpenCV adaptable to modern multi-core processors and GPU architectures, which are essential for computationally-intensive tasks, such as object detection and image recognition. -### What is KleidiCV? +## What is KleidiCV? KleidiCV is an Arm Kleidi Library, a suite of highly performant open-source Arm routines that leverages OpenCV’s HAL for hardware acceleration. KleidiCV focuses on optimizing various OpenCV functions specifically for Arm processors, ensuring faster execution and lower power consumption. This integration is particularly beneficial for mobile and embedded systems where performance efficiency is critical. By utilizing HAL, KleidiCV allows developers to harness the full potential of Arm hardware, making it an ideal solution for applications requiring high-performance computer vision on Arm-based devices. -### A Performant Solution +## A Performant Solution By combining OpenCV’s feature set, the hardware optimization from HAL, and specialized enhancements such as Arm’s KleidiCV, developers can build efficient, high-performing computer vision solutions tailored for modern mobile and embedded systems. In this Learning Path, you will learn how you can use KleidiCV-accelerated OpenCV in an Android application. diff --git a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md index 0c9741544c..2202b569db 100644 --- a/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md +++ b/content/learning-paths/mobile-graphics-and-gaming/android_opencv_kleidicv/process-images.md @@ -14,11 +14,11 @@ Start by adding an `assets` folder under `src/main`. Then, under the `assets` folder, add an `img.png` image file. This image can be any kind of image file, as the app will convert it to a specified image type through the image processing stage. -In this example, the image file is a [cameraman image](https://github.com/antimatter15/cameraman). +This Learning Path uses a [cameraman image](https://github.com/antimatter15/cameraman). -To facilitate easier navigation between files in Android Studio, use the **Project** menu option from the project browser pane. +For easier navigation between files in Android Studio, use the **Project** menu option from the project browser pane. -## ImageOperation +## `ImageOperation` You will now create an enum class, which is an enumeration, for a set of image processing operations in an application that uses the OpenCV library. In the `src/main/java/com/arm/arm64kleidicvdemo` file directory, add the `ImageOperation.kt` file, and modify it as follows: @@ -68,7 +68,9 @@ enum class ImageOperation(val displayName: String) { } ``` -The `ImageOperation` enum represents a collection of predefined image processing operations. Each enum constant is associated with a `displayName`, which is a user-friendly string describing the operation, and a unique implementation of the `apply` method to perform the operation on an image (Mat). +The `ImageOperation` enum represents a collection of predefined image processing operations. + +Each enum constant is associated with a `displayName`, which is a user-friendly string describing the operation, and a unique implementation of the `apply` method to perform the operation on an image (Mat). The processing results are available in the `dst` parameter of the `apply` method. @@ -81,11 +83,13 @@ Here we have four constants: To define its specific image processing logic, each enum constant must override the abstract method applied. -You have configured processing operations to align with current KleidiCV restrictions. Specifically, in-place changes are not supported, so the source and destination must be different images. Generally, only single-channel images are supported (Gaussian blur is an exception). Sobel’s output type must be 16SC1; dx and dy must be either (1,0) or (0,1); and the border mode must be replicated. Gaussian blur supports a non-zero sigma, but its performance is best with sigma 0.0. Its uplift is most noticeable with a kernel size of 7×7. +The processing operations defined here are written to align with the current KleidiCV specification. Specifically, in-place changes are not currently supported, so the source and destination images must be different. + +Generally, only single-channel images are supported; with Gaussian blur being an exception. Sobel’s output type must be 16SC1; dx and dy must be either (1,0) or (0,1); and the border mode must be replicated. Gaussian blur supports a non-zero sigma, but its performance is best with sigma 0.0. Its uplift is most noticeable with a kernel size of 7×7. There is also the companion object that provides a utility method `fromDisplayName`. This function maps the string `displayName` to its corresponding enum constant by iterating through the list of all enum values, and returns null if no match is found. -## ImageProcessor +## `ImageProcessor` Now add the `ImageProcessor.kt`: ```Kotlin @@ -100,13 +104,13 @@ class ImageProcessor { } ``` -The Kotlin code defines a simple utility class, `ImageProcessor`, which applies a specified image processing operation on an OpenCV Mat object. The `ImageProcessor` class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and efficient. +The Kotlin code defines a simple utility class; `ImageProcessor`, which applies a specified image processing operation on an OpenCV Mat object. The `ImageProcessor` class is a utility class that provides a method to process images. It does not store any state or have additional properties, making it lightweight and efficient. -The `ImageProcessor` class acts as a simple orchestrator for image processing tasks. It delegates the actual processing logic to the `ImageOperation` enum, which encapsulates various image processing techniques. This separation of concerns keeps the `ImageProcessor` class focused and makes it easy to extend or modify operations by updating the `ImageOperation` enum. +The `ImageProcessor` class acts as a simple orchestrator for image processing tasks. It delegates the actual processing logic to the `ImageOperation` enum, which encapsulates various image processing techniques. This separation of tasks ensures that the `ImageProcessor` class remains focused and it makes it easy to extend or modify operations by updating the `ImageOperation` enum. This design is clean and modular, allowing developers to easily add new processing operations or reuse the `ImageProcessor` in different parts of an application. It aligns with object-oriented principles by promoting encapsulation and reducing processing logic complexity. -## PerformanceMetrics +## `PerformanceMetrics` Now supplement the project with the `PerformanceMetrics.kt` file: ```Kotlin @@ -142,14 +146,12 @@ data class PerformanceMetrics(private val durationsNano: List) { } ``` -This code defines a `PerformanceMetrics` class that calculates and represents various statistical metrics for performance measurements, such as the average, minimum, maximum, and standard deviation of durations. - The `PerformanceMetrics` class analyzes and summarizes performance measurements, such as execution times or latency values. It accepts a list of durations in nanoseconds, computes key metrics like average, minimum, maximum, and standard deviation, and presents them in milliseconds. -By encapsulating the raw data `durationsNano` and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden `toString` method makes it easy to generate a human-readable summary for reporting or debugging purposes. Specifically, you will use this method to report the performance metrics to the user. +By encapsulating the raw data `durationsNano` and exposing only meaningful metrics through computed properties, the class ensures clear separation of data and functionality. The overridden `toString` method makes it easy to generate a human-readable summary for reporting or debugging purposes. You can use this method to report the performance metrics to the user. -## MainActivity -Finally, now modify `MainActivity.kt` as follows: +## `MainActivity` +You can now move on to modify `MainActivity.kt` as follows: ```Kotlin package com.arm.arm64kleidicvdemo @@ -301,7 +303,9 @@ class MainActivity : AppCompatActivity() { } ``` -The above Kotlin code defines the main activity for our application that demonstrates image processing using OpenCV. The `MainActivity` class extends `AppCompatActivity`, serving as the entry point for the app’s user interface. It manages the lifecycle of the activity and orchestrates the image processing logic. +This Kotlin code defines the main activity for our application. + +The `MainActivity` class extends `AppCompatActivity`, serving as the entry point for the app’s user interface. It manages the lifecycle of the activity and orchestrates the image processing logic. There are several members of this class: * `viewBinding` - manages UI components through the `ActivityMainBinding` class, simplifying access to views in the layout. @@ -328,7 +332,7 @@ The activity also implements several helper methods: 7. `measureOperationTime` - Measures the execution time of an operation in nanoseconds using System.nanoTime(). 8. `displayProcessedImage`. This method converts the processed Mat back to a Bitmap for display and updates the ImageView with the processed image. -## Databinding +## `Databinding` Finally, modify `build.gradle.kts` by adding the databinding under build features: ```JSON @@ -390,7 +394,9 @@ dependencies { ## Running the application You can now launch the application in an emulator, or on the actual device. -When you do so, click the **Load image** button, select the image processing operation, and then click **Process**. You will see the processing results and a detailed performance analysis as Figures 3-6 show. NOTE: This Learning Path used a Samsung Galaxy S22. +When you do so, click the **Load image** button, select the image processing operation, and then click **Process**. + +You will see the processing results and a detailed performance analysis as Figures 3-6 show. As a reminder, this Learning Path was tested on a Samsung Galaxy S22. ![img3](Figures/03.jpg) ![img4](Figures/04.jpg) @@ -398,14 +404,16 @@ When you do so, click the **Load image** button, select the image processing ope ![img6](Figures/06.jpg) ## Performance uplift -To appreciate the performance uplift offered by KleidiCV, now switch to one of the earliest OpenCV versions, which does not have KleidiCV. Version 4.9.0 is an example of one to try. +To appreciate the performance uplift offered by KleidiCV, now switch to one of the earliest OpenCV versions, which does not have KleidiCV. + +Version 4.9.0 is an example of one that works here. -To rerun the application open build.gradle.kts, and modify this line, from: +To rerun the application open `build.gradle.kts`, and modify this line, from: ```XML implementation("org.opencv:opencv:4.11.0") ``` -to: +To: ```XML implementation("org.opencv:opencv:4.9.0") ``` @@ -426,4 +434,6 @@ This particular example achieves the following performance uplift: | Resize | 0,02 | 0,04 | 2x | | Rotate | 0,02 | 0,06 | 3x | -As shown above, this has achieved a performance uplift of 4× faster computations for the Gaussian blur and the Sobel filter, 3× faster for rotation, and 2× faster for resizing. Note that these numbers are noise (large standard deviation), and measurements for another device might vary. \ No newline at end of file +As shown above, this has achieved a performance uplift of 4× faster computations for the Gaussian blur and the Sobel filter, 3× faster for rotation, and 2× faster for resizing. + +{{% notice Note %}} These numbers are noise (large standard deviation), and measurements for another device might vary.{{% /notice %}}