Fixes and improvements for StereoDepth, ColorCamera

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")  
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "8d27dfe16cbb72fa2649b87385a5a726127c4cfa")          
+set(DEPTHAI_DEVICE_SIDE_COMMIT "80b63be3ce97dc177df17865da85403ff08348ca")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/CMakeLists.txt

@@ -130,6 +130,9 @@ dai_add_example(device_queue_event src/device_queue_event_example.cpp)
 # OpenCV support example
 dai_add_example(opencv_support src/opencv_support_example.cpp)
 
+# RGB-depth alignment example
+dai_add_example(rgb_depth_aligned src/rgb_depth_aligned_example.cpp)
+
 # Device side decoding example for mobilenet-ssd
 dai_add_example(mobilenet_device_side_decoding src/mobilenet_device_side_decoding_example.cpp)
 target_compile_definitions(mobilenet_device_side_decoding PRIVATE BLOB_PATH="${mobilenet_blob}")

examples/src/rgb_depth_aligned_example.cpp

@@ -0,0 +1,110 @@
+#include <cstdio>
+#include <iostream>
+
+#include "utility.hpp"
+
+// Includes common necessary includes for development using depthai library
+#include "depthai/depthai.hpp"
+
+// Optional. If set (true), the ColorCamera is downscaled from 1080p to 720p.
+// Otherwise (false), the aligned depth is automatically upscaled to 1080p
+static constexpr bool downscaleColor = true;
+
+int main() {
+    using namespace std;
+
+    dai::Pipeline p;
+    std::vector<std::string> queueNames;
+
+    auto cam = p.create<dai::node::ColorCamera>();
+    cam->setBoardSocket(dai::CameraBoardSocket::RGB);
+    cam->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
+    if(downscaleColor) cam->setIspScale(2, 3);
+    // For now, RGB needs fixed focus to properly align with depth.
+    // This value was used during calibration
+    cam->initialControl.setManualFocus(130);
+
+    auto rgbOut = p.create<dai::node::XLinkOut>();
+    rgbOut->setStreamName("rgb");
+    queueNames.push_back("rgb");
+    cam->isp.link(rgbOut->input);
+
+    auto left = p.create<dai::node::MonoCamera>();
+    left->setResolution(dai::MonoCameraProperties::SensorResolution::THE_720_P);
+    left->setBoardSocket(dai::CameraBoardSocket::LEFT);
+
+    auto right = p.create<dai::node::MonoCamera>();
+    right->setResolution(dai::MonoCameraProperties::SensorResolution::THE_720_P);
+    right->setBoardSocket(dai::CameraBoardSocket::RIGHT);
+
+    auto stereo = p.create<dai::node::StereoDepth>();
+    stereo->setConfidenceThreshold(200);
+    // LR-check is required for depth alignment
+    stereo->setLeftRightCheck(true);
+    stereo->setDepthAlign(dai::CameraBoardSocket::RGB);
+    left->out.link(stereo->left);
+    right->out.link(stereo->right);
+
+    auto depthOut = p.create<dai::node::XLinkOut>();
+    depthOut->setStreamName("depth");
+    queueNames.push_back("depth");
+    // Currently we use the 'disparity' output. TODO 'depth'
+    stereo->disparity.link(depthOut->input);
+
+    // Connect to device
+    dai::Device d(p);
+    d.startPipeline();
+
+    // Sets queues size and behavior
+    for(const auto& name : queueNames) {
+        d.getOutputQueue(name, 4, false);
+    }
+
+    std::unordered_map<std::string, cv::Mat> frame;
+
+    while(1) {
+        std::unordered_map<std::string, std::shared_ptr<dai::ImgFrame>> latestPacket;
+
+        auto queueEvents = d.getQueueEvents(queueNames);
+        for(const auto& name : queueEvents) {
+            auto packets = d.getOutputQueue(name)->tryGetAll<dai::ImgFrame>();
+            auto count = packets.size();
+            if(count > 0) {
+                latestPacket[name] = packets[count - 1];
+            }
+        }
+
+        for(const auto& name : queueNames) {
+            if(latestPacket.find(name) != latestPacket.end()) {
+                if(name == "depth") {
+                    frame[name] = latestPacket[name]->getFrame();
+                    // Optional, extend range 0..95 -> 0..255, for a better visualisation
+                    if(1) frame[name].convertTo(frame[name], CV_8UC1, 255. / 95);
+                    // Optional, apply false colorization
+                    if(1) cv::applyColorMap(frame[name], frame[name], cv::COLORMAP_HOT);
+                } else {
+                    frame[name] = latestPacket[name]->getCvFrame();
+                }
+
+                cv::imshow(name, frame[name]);
+            }
+        }
+
+        // Blend when both received
+        if(frame.find("rgb") != frame.end() && frame.find("depth") != frame.end()) {
+            // Need to have both frames in BGR format before blending
+            if(frame["depth"].channels() < 3) {
+                cv::cvtColor(frame["depth"], frame["depth"], cv::COLOR_GRAY2BGR);
+            }
+            cv::Mat blended;
+            cv::addWeighted(frame["rgb"], 0.6, frame["depth"], 0.4, 0, blended);
+            cv::imshow("rgb-depth", blended);
+            frame.clear();
+        }
+
+        int key = cv::waitKey(1);
+        if(key == 'q') {
+            return 0;
+        }
+    }
+}

include/depthai/pipeline/datatype/CameraControl.hpp

@@ -105,10 +105,10 @@ class CameraControl : public Buffer {
     void setAutoExposureRegion(uint16_t startX, uint16_t startY, uint16_t width, uint16_t height);
 
     /**
-     * Set a command to specify auto exposure compenstaion
-     * @param compensation Compensation value between -128..127
+     * Set a command to specify auto exposure compensation
+     * @param compensation Compensation value between -9..9
      */
-    void setAutoExposureCompensation(int8_t compensation);
+    void setAutoExposureCompensation(int compensation);
 
     /**
      * Set a command to specify auto banding mode
@@ -119,7 +119,7 @@ class CameraControl : public Buffer {
     /**
      * Set a command to manually specify exposure
      * @param exposureTimeUs Exposure time in microseconds
-     * @param sensitivityIso Sensitivity as ISO value
+     * @param sensitivityIso Sensitivity as ISO value, usual range 100..1600
      */
     void setManualExposure(uint32_t exposureTimeUs, uint32_t sensitivityIso);
 
@@ -138,46 +138,40 @@ class CameraControl : public Buffer {
 
     // Other image controls
     /**
-     * Set a command to specify auto white balance lock
-     * @param lock Auto white balance lock mode enabled or disabled
-     */
-    void setBrightness(uint16_t value);  // TODO move to AE?
-
-    /**
-     * Set a command to specify auto white balance lock
-     * @param lock Auto white balance lock mode enabled or disabled
+     * Set a command to adjust image brightness
+     * @param value Brightness, range -10..10
      */
-    void setContrast(uint16_t value);
+    void setBrightness(int value);
 
     /**
-     * Set a command to specify saturation value
-     * @param value Saturation
+     * Set a command to adjust image contrast
+     * @param value Contrast, range -10..10
      */
-    void setSaturation(uint16_t value);
+    void setContrast(int value);
 
     /**
-     * Set a command to specify sharpness value
-     * @param value Sharpness
+     * Set a command to adjust image saturation
+     * @param value Saturation, range -10..10
      */
-    void setSharpness(uint16_t value);
+    void setSaturation(int value);
 
     /**
-     * Set a command to specify noise reduction strength
-     * @param value Noise reduction strength
+     * Set a command to adjust image sharpness
+     * @param value Sharpness, range 0..4
      */
-    void setNoiseReductionStrength(uint16_t value);
+    void setSharpness(int value);
 
     /**
-     * Set a command to specify luma denoise value
-     * @param value Luma denoise
+     * Set a command to adjust luma denoise amount
+     * @param value Luma denoise amount, range 0..4
      */
-    void setLumaDenoise(uint16_t value);
+    void setLumaDenoise(int value);
 
     /**
-     * Set a command to specify chroma denoise value
-     * @param value Chroma denoise
+     * Set a command to adjust chroma denoise amount
+     * @param value Chroma denoise amount, range 0..4
      */
-    void setChromaDenoise(uint16_t value);
+    void setChromaDenoise(int value);
 
     /**
      * Set a command to specify scene mode

include/depthai/pipeline/node/ColorCamera.hpp

@@ -25,6 +25,8 @@ class ColorCamera : public Node {
 
     std::shared_ptr<RawCameraControl> rawControl;
 
+    int getScaledSize(int input, int num, int denom) const;
+
    public:
     /**
      * Constructs ColorCamera node.
@@ -71,6 +73,20 @@ class ColorCamera : public Node {
      */
     Output still{*this, "still", Output::Type::MSender, {{DatatypeEnum::ImgFrame, false}}};
 
+    /**
+     * Outputs ImgFrame message that carries YUV420 planar (I420/IYUV) frame data.
+     *
+     * Generated by the ISP engine, and the source for the 'video', 'preview' and 'still' outputs
+     */
+    Output isp{*this, "isp", Output::Type::MSender, {{DatatypeEnum::ImgFrame, false}}};
+
+    /**
+     * Outputs ImgFrame message that carries RAW10-packed (MIPI CSI-2 format) frame data.
+     *
+     * Captured directly from the camera sensor, and the source for the 'isp' output.
+     */
+    Output raw{*this, "raw", Output::Type::MSender, {{DatatypeEnum::ImgFrame, false}}};
+
     /**
      * Specify which board socket to use
      * @param boardSocket Board socket to use
@@ -116,18 +132,47 @@ class ColorCamera : public Node {
     /// Set preview output size
     void setPreviewSize(int width, int height);
 
+    /// Set preview output size, as a tuple <width, height>
+    void setPreviewSize(std::tuple<int, int> size);
+
     /// Set video output size
     void setVideoSize(int width, int height);
 
+    /// Set video output size, as a tuple <width, height>
+    void setVideoSize(std::tuple<int, int> size);
+
     /// Set still output size
     void setStillSize(int width, int height);
 
+    /// Set still output size, as a tuple <width, height>
+    void setStillSize(std::tuple<int, int> size);
+
     /// Set sensor resolution
     void setResolution(Properties::SensorResolution resolution);
 
     /// Get sensor resolution
     Properties::SensorResolution getResolution() const;
 
+    /**
+     * Set 'isp' output scaling (numerator/denominator), preserving the aspect ratio.
+     * @param numerator Range 1..16
+     * @param denominator Range 1..63
+     */
+    void setIspScale(int numerator, int denominator);
+
+    /// Set 'isp' output scaling, as a tuple <numerator, denominator>
+    void setIspScale(std::tuple<int, int> scale);
+
+    /**
+     * Set 'isp' output scaling, per each direction. If the horizontal scaling factor
+     * (horizNum/horizDen) is different than the vertical scaling factor
+     * (vertNum/vertDen), a distorted (stretched or squished) image is generated
+     */
+    void setIspScale(int horizNum, int horizDenom, int vertNum, int vertDenom);
+
+    /// Set 'isp' output scaling, per each direction, as <numerator, denominator> tuples
+    void setIspScale(std::tuple<int, int> horizScale, std::tuple<int, int> vertScale);
+
     /**
      * Set rate at which camera should produce frames
      * @param fps Rate in frames per second
@@ -168,6 +213,13 @@ class ColorCamera : public Node {
     /// Get sensor resolution height
     int getResolutionHeight() const;
 
+    /// Get 'isp' output resolution as size, after scaling
+    std::tuple<int, int> getIspSize() const;
+    /// Get 'isp' output width
+    int getIspWidth() const;
+    /// Get 'isp' output height
+    int getIspHeight() const;
+
     /**
      * Specify sensor center crop.
      * Resolution size / video size

include/depthai/pipeline/node/StereoDepth.hpp

@@ -98,6 +98,17 @@ class StereoDepth : public Node {
      */
     void setMedianFilter(Properties::MedianFilter median);
 
+    /**
+     * @param align Set the disparity/depth alignment: centered (between the 'left' and 'right' inputs),
+     * or from the perspective of a rectified output stream
+     */
+    void setDepthAlign(Properties::DepthAlign align);
+
+    /**
+     * @param camera Set the camera from whose perspective the disparity/depth will be aligned
+     */
+    void setDepthAlign(CameraBoardSocket camera);
+
     /**
      * Confidence threshold for disparity calculation
      * @param confThr Confidence threshold value 0..255
@@ -138,15 +149,17 @@ class StereoDepth : public Node {
     void setRectifyMirrorFrame(bool enable);
 
     /**
-     * Enable outputting rectified frames. Optimizes computation on device side when disabled
+     * Enable outputting rectified frames. Optimizes computation on device side when disabled.
+     * DEPRECATED. The outputs are auto-enabled if used
      */
-    void setOutputRectified(bool enable);
+    [[deprecated("Function call should be removed")]] void setOutputRectified(bool enable);
 
     /**
      * Enable outputting 'depth' stream (converted from disparity).
-     * In certain configurations, this will disable 'disparity' stream
+     * In certain configurations, this will disable 'disparity' stream.
+     * DEPRECATED. The output is auto-enabled if used
      */
-    void setOutputDepth(bool enable);
+    [[deprecated("Function call should be removed")]] void setOutputDepth(bool enable);
 };
 
 }  // namespace node

src/opencv/ImgFrame.cpp

@@ -118,7 +118,7 @@ cv::Mat ImgFrame::getCvFrame() {
         } break;
 
         case Type::YUV420p:
-            cv::cvtColor(frame, output, cv::ColorConversionCodes::COLOR_YUV420p2BGR);
+            cv::cvtColor(frame, output, cv::ColorConversionCodes::COLOR_YUV2BGR_IYUV);
             break;
 
         case Type::NV12: