Fixes and improvements for StereoDepth, ColorCamera

.github/workflows/main.workflow.yml

@@ -57,7 +57,7 @@ jobs:
       run: cmake . -Bbuild -DDEPTHAI_CLANG_TIDY=ON -DCLANG_TIDY_BIN=clang-tidy-10
 
     - name: Run clang-tidy
-      run: cmake --build build --parallel
+      run: cmake --build build --parallel 8
 
 
   build:
@@ -105,7 +105,7 @@ jobs:
       run: cmake -S . -B build -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON
 
     - name: build
-      run: cmake --build build --parallel
+      run: cmake --build build --parallel 8
 
   integration: 
     runs-on: ${{ matrix.os }}
@@ -162,26 +162,26 @@ jobs:
       run: echo "$GITHUB_WORKSPACE/depthai_install/bin/" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
     - name: Build and install
-      run: cmake --build build --config ${{ matrix.build-type }} --target install --parallel
+      run: cmake --build build --config ${{ matrix.build-type }} --target install --parallel 8
 
     - name: Build and test add_subdirectory
       run: |
         cmake -S tests/integration/ -B tests/integration/build_add_subdirectory -D TEST_FIND_PACKAGE=OFF
-        cmake --build tests/integration/build_add_subdirectory --config ${{ matrix.build-type }} --parallel
+        cmake --build tests/integration/build_add_subdirectory --config ${{ matrix.build-type }} --parallel 8
         cd tests/integration/build_add_subdirectory
         ctest -C ${{ matrix.build-type }}
 
     - name: Build and test find_package (installed)
       run: |
         cmake -S tests/integration/ -B tests/integration/build_find_package -D TEST_FIND_PACKAGE=ON -D depthai_DIR=$GITHUB_WORKSPACE/depthai_install/lib/cmake/depthai
-        cmake --build tests/integration/build_find_package --config ${{ matrix.build-type }} --parallel
+        cmake --build tests/integration/build_find_package --config ${{ matrix.build-type }} --parallel 8
         cd tests/integration/build_find_package
         ctest -C ${{ matrix.build-type }}
 
     - name: Build and test find_package (build directory)
       run: |
         cmake -S tests/integration/ -B tests/integration/build_find_package_2 -D TEST_FIND_PACKAGE=ON -D depthai_DIR=$GITHUB_WORKSPACE/build
-        cmake --build tests/integration/build_find_package_2 --config ${{ matrix.build-type }} --parallel
+        cmake --build tests/integration/build_find_package_2 --config ${{ matrix.build-type }} --parallel 8
         cd tests/integration/build_find_package_2
         ctest -C ${{ matrix.build-type }}
 

README.md

@@ -48,22 +48,22 @@ git submodule update --init --recursive
 ```
 mkdir build && cd build
 cmake ..
-cmake --build . --parallel
+cmake --build . --parallel 8
 ```
 
 **Dynamic library**
 ```
 mkdir build && cd build
 cmake .. -D BUILD_SHARED_LIBS=ON
-cmake --build . --parallel
+cmake --build . --parallel 8
 ```
 ## Installing
 
 To install specify optional prefix and build target install
 ```
 cmake .. -D CMAKE_INSTALL_PREFIX=[path/to/install/dir]
-cmake --build . --parallel
-cmake --build . --target install --parallel
+cmake --build . --parallel 8
+cmake --build . --target install --parallel 8
 ```
 
 ## Running tests
@@ -72,7 +72,7 @@ To run the tests build the library with the following options
 ```
 mkdir build_tests && cd build_tests
 cmake .. -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_BUILD_EXAMPLES=ON
-cmake --build . --parallel
+cmake --build . --parallel 8
 ctest
 ```
 

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")  
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "8d27dfe16cbb72fa2649b87385a5a726127c4cfa")          
+set(DEPTHAI_DEVICE_SIDE_COMMIT "80b63be3ce97dc177df17865da85403ff08348ca")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/CMakeLists.txt

@@ -130,6 +130,9 @@ dai_add_example(device_queue_event src/device_queue_event_example.cpp)
 # OpenCV support example
 dai_add_example(opencv_support src/opencv_support_example.cpp)
 
+# RGB-depth alignment example
+dai_add_example(rgb_depth_aligned src/rgb_depth_aligned_example.cpp)
+
 # Device side decoding example for mobilenet-ssd
 dai_add_example(mobilenet_device_side_decoding src/mobilenet_device_side_decoding_example.cpp)
 target_compile_definitions(mobilenet_device_side_decoding PRIVATE BLOB_PATH="${mobilenet_blob}")

examples/src/rgb_depth_aligned_example.cpp

@@ -0,0 +1,110 @@
+#include <cstdio>
+#include <iostream>
+
+#include "utility.hpp"
+
+// Includes common necessary includes for development using depthai library
+#include "depthai/depthai.hpp"
+
+// Optional. If set (true), the ColorCamera is downscaled from 1080p to 720p.
+// Otherwise (false), the aligned depth is automatically upscaled to 1080p
+static constexpr bool downscaleColor = true;
+
+int main() {
+    using namespace std;
+
+    dai::Pipeline p;
+    std::vector<std::string> queueNames;
+
+    auto cam = p.create<dai::node::ColorCamera>();
+    cam->setBoardSocket(dai::CameraBoardSocket::RGB);
+    cam->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
+    if(downscaleColor) cam->setIspScale(2, 3);
+    // For now, RGB needs fixed focus to properly align with depth.
+    // This value was used during calibration
+    cam->initialControl.setManualFocus(130);
+
+    auto rgbOut = p.create<dai::node::XLinkOut>();
+    rgbOut->setStreamName("rgb");
+    queueNames.push_back("rgb");
+    cam->isp.link(rgbOut->input);
+
+    auto left = p.create<dai::node::MonoCamera>();
+    left->setResolution(dai::MonoCameraProperties::SensorResolution::THE_720_P);
+    left->setBoardSocket(dai::CameraBoardSocket::LEFT);
+
+    auto right = p.create<dai::node::MonoCamera>();
+    right->setResolution(dai::MonoCameraProperties::SensorResolution::THE_720_P);
+    right->setBoardSocket(dai::CameraBoardSocket::RIGHT);
+
+    auto stereo = p.create<dai::node::StereoDepth>();
+    stereo->setConfidenceThreshold(200);
+    // LR-check is required for depth alignment
+    stereo->setLeftRightCheck(true);
+    stereo->setDepthAlign(dai::CameraBoardSocket::RGB);
+    left->out.link(stereo->left);
+    right->out.link(stereo->right);
+
+    auto depthOut = p.create<dai::node::XLinkOut>();
+    depthOut->setStreamName("depth");
+    queueNames.push_back("depth");
+    // Currently we use the 'disparity' output. TODO 'depth'
+    stereo->disparity.link(depthOut->input);
+
+    // Connect to device
+    dai::Device d(p);
+    d.startPipeline();
+
+    // Sets queues size and behavior
+    for(const auto& name : queueNames) {
+        d.getOutputQueue(name, 4, false);
+    }
+
+    std::unordered_map<std::string, cv::Mat> frame;
+
+    while(1) {
+        std::unordered_map<std::string, std::shared_ptr<dai::ImgFrame>> latestPacket;
+
+        auto queueEvents = d.getQueueEvents(queueNames);
+        for(const auto& name : queueEvents) {
+            auto packets = d.getOutputQueue(name)->tryGetAll<dai::ImgFrame>();
+            auto count = packets.size();
+            if(count > 0) {
+                latestPacket[name] = packets[count - 1];
+            }
+        }
+
+        for(const auto& name : queueNames) {
+            if(latestPacket.find(name) != latestPacket.end()) {
+                if(name == "depth") {
+                    frame[name] = latestPacket[name]->getFrame();
+                    // Optional, extend range 0..95 -> 0..255, for a better visualisation
+                    if(1) frame[name].convertTo(frame[name], CV_8UC1, 255. / 95);
+                    // Optional, apply false colorization
+                    if(1) cv::applyColorMap(frame[name], frame[name], cv::COLORMAP_HOT);
+                } else {
+                    frame[name] = latestPacket[name]->getCvFrame();
+                }
+
+                cv::imshow(name, frame[name]);
+            }
+        }
+
+        // Blend when both received
+        if(frame.find("rgb") != frame.end() && frame.find("depth") != frame.end()) {
+            // Need to have both frames in BGR format before blending
+            if(frame["depth"].channels() < 3) {
+                cv::cvtColor(frame["depth"], frame["depth"], cv::COLOR_GRAY2BGR);
+            }
+            cv::Mat blended;
+            cv::addWeighted(frame["rgb"], 0.6, frame["depth"], 0.4, 0, blended);
+            cv::imshow("rgb-depth", blended);
+            frame.clear();
+        }
+
+        int key = cv::waitKey(1);
+        if(key == 'q') {
+            return 0;
+        }
+    }
+}

examples/src/spatial_location_calculator_example.cpp

@@ -35,14 +35,10 @@ int main() {
     monoRight->setResolution(dai::MonoCameraProperties::SensorResolution::THE_400_P);
     monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT);
 
-    bool outputDepth = true;
-    bool outputRectified = false;
     bool lrcheck = false;
     bool subpixel = false;
 
     // StereoDepth
-    stereo->setOutputDepth(outputDepth);
-    stereo->setOutputRectified(outputRectified);
     stereo->setConfidenceThreshold(255);
 
     // stereo->setMedianFilter(dai::StereoDepthProperties::MedianFilter::MEDIAN_OFF);

examples/src/spatial_mobilenet_example.cpp

@@ -45,7 +45,6 @@ dai::Pipeline createNNPipeline(std::string nnPath) {
     monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT);
 
     /// setting node configs
-    stereo->setOutputDepth(true);
     stereo->setConfidenceThreshold(255);
 
     spatialDetectionNetwork->setBlobPath(nnPath);
@@ -189,4 +188,4 @@ int main(int argc, char** argv) {
     }
 
     return 0;
-}
+}

examples/src/spatial_mobilenet_mono_example.cpp

@@ -43,8 +43,6 @@ dai::Pipeline createNNPipeline(std::string nnPath) {
     monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT);
 
     // StereoDepth
-    stereo->setOutputDepth(true);
-    stereo->setOutputRectified(true);
     stereo->setConfidenceThreshold(255);
 
     // Link plugins CAM -> STEREO -> XLINK
@@ -199,4 +197,4 @@ int main(int argc, char** argv) {
     }
 
     return 0;
-}
+}

examples/src/spatial_tiny_yolo_example.cpp

@@ -52,7 +52,6 @@ dai::Pipeline createNNPipeline(std::string nnPath) {
     monoRight->setBoardSocket(dai::CameraBoardSocket::RIGHT);
 
     /// setting node configs
-    stereo->setOutputDepth(true);
     stereo->setConfidenceThreshold(255);
 
     spatialDetectionNetwork->setBlobPath(nnPath);
@@ -203,4 +202,4 @@ int main(int argc, char** argv) {
     }
 
     return 0;
-}
+}

examples/src/stereo_example.cpp

@@ -56,8 +56,6 @@ int main() {
 
     if(withDepth) {
         // StereoDepth
-        stereo->setOutputDepth(outputDepth);
-        stereo->setOutputRectified(outputRectified);
         stereo->setConfidenceThreshold(200);
         stereo->setRectifyEdgeFillColor(0);  // black, to better see the cutout
         // stereo->loadCalibrationFile("../../../../depthai/resources/depthai.calib");

include/depthai/pipeline/datatype/CameraControl.hpp

@@ -105,10 +105,10 @@ class CameraControl : public Buffer {
     void setAutoExposureRegion(uint16_t startX, uint16_t startY, uint16_t width, uint16_t height);
 
     /**
-     * Set a command to specify auto exposure compenstaion
-     * @param compensation Compensation value between -128..127
+     * Set a command to specify auto exposure compensation
+     * @param compensation Compensation value between -9..9
      */
-    void setAutoExposureCompensation(int8_t compensation);
+    void setAutoExposureCompensation(int compensation);
 
     /**
      * Set a command to specify auto banding mode
@@ -119,7 +119,7 @@ class CameraControl : public Buffer {
     /**
      * Set a command to manually specify exposure
      * @param exposureTimeUs Exposure time in microseconds
-     * @param sensitivityIso Sensitivity as ISO value
+     * @param sensitivityIso Sensitivity as ISO value, usual range 100..1600
      */
     void setManualExposure(uint32_t exposureTimeUs, uint32_t sensitivityIso);
 
@@ -138,46 +138,40 @@ class CameraControl : public Buffer {
 
     // Other image controls
     /**
-     * Set a command to specify auto white balance lock
-     * @param lock Auto white balance lock mode enabled or disabled
-     */
-    void setBrightness(uint16_t value);  // TODO move to AE?
-
-    /**
-     * Set a command to specify auto white balance lock
-     * @param lock Auto white balance lock mode enabled or disabled
+     * Set a command to adjust image brightness
+     * @param value Brightness, range -10..10
      */
-    void setContrast(uint16_t value);
+    void setBrightness(int value);
 
     /**
-     * Set a command to specify saturation value
-     * @param value Saturation
+     * Set a command to adjust image contrast
+     * @param value Contrast, range -10..10
      */
-    void setSaturation(uint16_t value);
+    void setContrast(int value);
 
     /**
-     * Set a command to specify sharpness value
-     * @param value Sharpness
+     * Set a command to adjust image saturation
+     * @param value Saturation, range -10..10
      */
-    void setSharpness(uint16_t value);
+    void setSaturation(int value);
 
     /**
-     * Set a command to specify noise reduction strength
-     * @param value Noise reduction strength
+     * Set a command to adjust image sharpness
+     * @param value Sharpness, range 0..4
      */
-    void setNoiseReductionStrength(uint16_t value);
+    void setSharpness(int value);
 
     /**
-     * Set a command to specify luma denoise value
-     * @param value Luma denoise
+     * Set a command to adjust luma denoise amount
+     * @param value Luma denoise amount, range 0..4
      */
-    void setLumaDenoise(uint16_t value);
+    void setLumaDenoise(int value);
 
     /**
-     * Set a command to specify chroma denoise value
-     * @param value Chroma denoise
+     * Set a command to adjust chroma denoise amount
+     * @param value Chroma denoise amount, range 0..4
      */
-    void setChromaDenoise(uint16_t value);
+    void setChromaDenoise(int value);
 
     /**
      * Set a command to specify scene mode