Gen2: ImageManip rotated/arbitrary cropping, camera controls

include/depthai-shared/datatype/RawCameraControl.hpp

@@ -9,16 +9,258 @@
 namespace dai {
 
 struct RawCameraControl : public RawBuffer {
-    // TODO(themarpe/alex) - this is a placeholder for vairous camera control commands
-    bool captureStill = false;
+    enum class Command : uint8_t {
+        START_STREAM = 1,
+        STOP_STREAM = 2,
+        STILL_CAPTURE = 3,
+        MOVE_LENS = 4, /* [1] lens position: 0-255
+                        */
+        AF_TRIGGER = 5,
+        AE_MANUAL = 6, /* [1] exposure time [us]
+                        * [2] sensitivity [iso]
+                        * [3] frame duration [us]
+                        */
+        AE_AUTO = 7,
+        AWB_MODE = 8,                  /* [1] awb_mode: AutoWhiteBalanceMode
+                                        */
+        SCENE_MODE = 9,                /* [1] scene_mode: SceneMode
+                                        */
+        ANTIBANDING_MODE = 10,         /* [1] antibanding_mode: AntiBandingMode
+                                        */
+        EXPOSURE_COMPENSATION = 11,    /* [1] value
+                                        */
+        AE_LOCK = 13,                  /* [1] ae_lock_mode: bool
+                                        */
+        AE_TARGET_FPS_RANGE = 14,      /* [1] min_fps
+                                        * [2] max_fps
+                                        */
+        AWB_LOCK = 16,                 /* [1] awb_lock_mode: bool
+                                        */
+        CAPTURE_INTENT = 17,           /* [1] capture_intent_mode: CaptureIntent
+                                        */
+        CONTROL_MODE = 18,             /* [1] control_mode: ControlMode
+                                        */
+        FRAME_DURATION = 21,           /* [1] frame_duration
+                                        */
+        SENSITIVITY = 23,              /* [1] iso_val
+                                        */
+        EFFECT_MODE = 24,              /* [1] effect_mode: EffectMode
+                                        */
+        AF_MODE = 26,                  /* [1] af_mode: AutoFocusMode
+                                        */
+        NOISE_REDUCTION_STRENGTH = 27, /* [1] value
+                                        */
+        SATURATION = 28,               /* [1] value
+                                        */
+        BRIGHTNESS = 31,               /* [1] value
+                                        */
+        STREAM_FORMAT = 33,            /* [1] format
+                                        */
+        RESOLUTION = 34,               /* [1] width
+                                        * [2] height
+                                        */
+        SHARPNESS = 35,                /* [1] value
+                                        */
+        CUSTOM_USECASE = 40,           /* [1] value
+                                        */
+        CUSTOM_CAPT_MODE = 41,         /* [1] value
+                                        */
+        CUSTOM_EXP_BRACKETS = 42,      /* [1] val1
+                                        * [2] val2
+                                        * [3] val3
+                                        */
+        CUSTOM_CAPTURE = 43,           /* [1] value
+                                        */
+        CONTRAST = 44,                 /* [1] value
+                                        */
+        AE_REGION = 45,                /* [1] x
+                                        * [2] y
+                                        * [3] width
+                                        * [4] height
+                                        * [5] priority
+                                        */
+        AF_REGION = 46,                /* [1] x
+                                        * [2] y
+                                        * [3] width
+                                        * [4] height
+                                        * [5] priority
+                                        */
+        LUMA_DENOISE = 47,             /* [1] value
+                                        */
+        CHROMA_DENOISE = 48,           /* [1] value
+                                        */
+    };
+
+    enum class AutoFocusMode : uint8_t {
+        /// Autofocus disabled. Suitable for manual focus
+        OFF = 0,
+        /// Runs autofocus once at startup, and at subsequent trigger commands
+        AUTO,
+        /// TODO
+        MACRO,
+        /// Runs autofocus when the scene is detected as out-of-focus
+        CONTINUOUS_VIDEO,
+        CONTINUOUS_PICTURE,
+        EDOF,
+    };
+
+    enum class AutoWhiteBalanceMode : uint8_t {
+        OFF = 0,
+        AUTO,
+        INCANDESCENT,
+        FLUORESCENT,
+        WARM_FLUORESCENT,
+        DAYLIGHT,
+        CLOUDY_DAYLIGHT,
+        TWILIGHT,
+        SHADE,
+    };
+
+    enum class SceneMode : uint8_t {
+        UNSUPPORTED = 0,
+        FACE_PRIORITY,
+        ACTION,
+        PORTRAIT,
+        LANDSCAPE,
+        NIGHT,
+        NIGHT_PORTRAIT,
+        THEATRE,
+        BEACH,
+        SNOW,
+        SUNSET,
+        STEADYPHOTO,
+        FIREWORKS,
+        SPORTS,
+        PARTY,
+        CANDLELIGHT,
+        BARCODE,
+    };
+
+    enum class AntiBandingMode : uint8_t {
+        OFF = 0,
+        _50HZ,
+        _60HZ,
+        AUTO,
+    };
+
+    enum class CaptureIntent : uint8_t {
+        CUSTOM = 0,
+        PREVIEW,
+        STILL_CAPTURE,
+        VIDEO_RECORD,
+        VIDEO_SNAPSHOT,
+        ZERO_SHUTTER_LAG,
+    };
+
+    enum class ControlMode : uint8_t {
+        OFF = 0,
+        AUTO,
+        USE_SCENE_MODE,
+    };
+
+    enum class EffectMode : uint8_t {
+        OFF = 0,
+        MONO,
+        NEGATIVE,
+        SOLARIZE,
+        SEPIA,
+        POSTERIZE,
+        WHITEBOARD,
+        BLACKBOARD,
+        AQUA,
+    };
+
+    struct ManualExposureParams {
+        uint32_t exposureTimeUs;
+        uint32_t sensitivityIso;
+        uint32_t frameDurationUs;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(ManualExposureParams, exposureTimeUs, sensitivityIso, frameDurationUs);
+    };
+
+    // AE_REGION / AF_REGION
+    struct RegionParams {
+        uint16_t x;
+        uint16_t y;
+        uint16_t width;
+        uint16_t height;
+        // Set to 1 for now. TODO
+        uint32_t priority;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(RegionParams, x, y, width, height, priority);
+    };
+
+    uint64_t cmdMask = 0;
+
+    AutoFocusMode autoFocusMode = AutoFocusMode::CONTINUOUS_VIDEO;
+
+    /**
+     * Lens/VCM position, range: 0..255. Used with `autoFocusMode = OFF`.
+     * With current IMX378 modules:
+     * - max 255: macro focus, at 8cm distance
+     * - infinite focus at about 120..130 (may vary from module to module)
+     * - lower values lead to out-of-focus (lens too close to the sensor array)
+     */
+    uint8_t lensPosition = 0;
+
+    ManualExposureParams expManual;
+    RegionParams aeRegion, afRegion;
+    AutoWhiteBalanceMode awbMode;
+    SceneMode sceneMode;
+    AntiBandingMode antiBandingMode;
+    bool aeLockMode;
+    bool awbLockMode;
+    EffectMode effectMode;
+    int8_t expCompensation;
+    uint16_t brightness;
+    uint16_t contrast;
+    uint16_t saturation;
+    uint16_t sharpness;
+    uint16_t noiseReductionStrength;
+    uint16_t lumaDenoise;
+    uint16_t chromaDenoise;
+
+    void setCommand(Command cmd, bool value = true) {
+        uint64_t mask = 1ull << (uint8_t)cmd;
+        if(value)
+            cmdMask |= mask;
+        else
+            cmdMask &= ~mask;
+    }
+    void clearCommand(Command cmd) {
+        setCommand(cmd, false);
+    }
+    bool getCommand(Command cmd) {
+        return !!(cmdMask & (1ull << (uint8_t)cmd));
+    }
 
     void serialize(std::vector<std::uint8_t>& metadata, DatatypeEnum& datatype) override {
         nlohmann::json j = *this;
         metadata = nlohmann::json::to_msgpack(j);
         datatype = DatatypeEnum::CameraControl;
     };
 
-    NLOHMANN_DEFINE_TYPE_INTRUSIVE(RawCameraControl, captureStill);
+    NLOHMANN_DEFINE_TYPE_INTRUSIVE(RawCameraControl,
+                                   cmdMask,
+                                   autoFocusMode,
+                                   lensPosition,
+                                   expManual,
+                                   aeRegion,
+                                   afRegion,
+                                   awbMode,
+                                   sceneMode,
+                                   antiBandingMode,
+                                   aeLockMode,
+                                   awbLockMode,
+                                   effectMode,
+                                   expCompensation,
+                                   brightness,
+                                   contrast,
+                                   saturation,
+                                   sharpness,
+                                   noiseReductionStrength,
+                                   lumaDenoise,
+                                   chromaDenoise);
 };
 
-}  // namespace dai
+}  // namespace dai

include/depthai-shared/datatype/RawImageManipConfig.hpp

@@ -18,21 +18,81 @@ struct RawImageManipConfig : public RawBuffer {
         NLOHMANN_DEFINE_TYPE_INTRUSIVE(CropRect, xmin, ymin, xmax, ymax);
     };
 
+    // Point2f, Size2f, RotatedRect are defined similar to OpenCV types
+    struct Point2f {
+        float x;
+        float y;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(Point2f, x, y);
+    };
+
+    struct Size2f {
+        float width;
+        float height;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(Size2f, width, height);
+    };
+
+    struct RotatedRect {
+        Point2f center;
+        Size2f size;
+        // degrees, increasing clockwise
+        float angle;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(RotatedRect, center, size, angle);
+    };
+
     struct CropConfig {
         CropRect cropRect;
+        RotatedRect cropRotatedRect;
+
         bool enableCenterCropRectangle = false;
         // if enableCenterCropRectangle -> automatically calculated crop parameters
         float cropRatio = 1.0f, widthHeightAspectRatio = 1.0f;
 
-        NLOHMANN_DEFINE_TYPE_INTRUSIVE(CropConfig, cropRect, enableCenterCropRectangle, cropRatio, widthHeightAspectRatio);
+        bool enableRotatedRect = false;
+
+        // Range 0..1 by default. Set 'false' to specify in pixels
+        bool normalizedCoords = true;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(
+            CropConfig, cropRect, cropRotatedRect, enableCenterCropRectangle, cropRatio, widthHeightAspectRatio, enableRotatedRect, normalizedCoords);
     };
 
     struct ResizeConfig {
         int width = 0, height = 0;
         bool lockAspectRatioFill = false;
         char bgRed = 0, bgGreen = 0, bgBlue = 0;
 
-        NLOHMANN_DEFINE_TYPE_INTRUSIVE(ResizeConfig, width, height, lockAspectRatioFill, bgRed, bgGreen, bgBlue);
+        //  clockwise order, pt[0] is mapped to the top-left output corner
+        std::vector<Point2f> warpFourPoints;
+        bool normalizedCoords = true;
+        bool enableWarp4pt = false;
+
+        std::vector<float> warpMatrix3x3;
+        bool enableWarpMatrix = false;
+
+        // clockwise
+        float rotationAngle;
+        // false -> degrees
+        bool angleInRadians = false;
+        bool enableRotation = false;
+
+        NLOHMANN_DEFINE_TYPE_INTRUSIVE(ResizeConfig,
+                                       width,
+                                       height,
+                                       lockAspectRatioFill,
+                                       bgRed,
+                                       bgGreen,
+                                       bgBlue,
+                                       warpFourPoints,
+                                       normalizedCoords,
+                                       enableWarp4pt,
+                                       warpMatrix3x3,
+                                       enableWarpMatrix,
+                                       rotationAngle,
+                                       angleInRadians,
+                                       enableRotation);
     };
 
     struct FormatConfig {
@@ -50,13 +110,19 @@ struct RawImageManipConfig : public RawBuffer {
     bool enableResize = false;
     bool enableFormat = false;
 
+    // Usable with runtime config only,
+    // when ImageManipProperties.inputConfigSync is set
+    bool reusePreviousImage = false;
+    bool skipCurrentImage = false;
+
     void serialize(std::vector<std::uint8_t>& metadata, DatatypeEnum& datatype) override {
         nlohmann::json j = *this;
         metadata = nlohmann::json::to_msgpack(j);
         datatype = DatatypeEnum::ImageManipConfig;
     };
 
-    NLOHMANN_DEFINE_TYPE_INTRUSIVE(RawImageManipConfig, cropConfig, resizeConfig, formatConfig, enableCrop, enableResize, enableFormat);
+    NLOHMANN_DEFINE_TYPE_INTRUSIVE(
+        RawImageManipConfig, cropConfig, resizeConfig, formatConfig, enableCrop, enableResize, enableFormat, reusePreviousImage, skipCurrentImage);
 };
 
-}  // namespace dai
+}  // namespace dai

include/depthai-shared/properties/ColorCameraProperties.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <depthai-shared/datatype/RawCameraControl.hpp>
 #include <nlohmann/json.hpp>
 
 #include "depthai-shared/common/CameraBoardSocket.hpp"
@@ -23,6 +24,8 @@ struct ColorCameraProperties {
      */
     enum class ColorOrder : int32_t { BGR, RGB };
 
+    RawCameraControl initialConfig;
+
     /**
      * Which socket will color camera use
      */
@@ -102,6 +105,7 @@ struct ColorCameraProperties {
 };
 
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ColorCameraProperties,
+                                   initialConfig,
                                    boardSocket,
                                    imageOrientation,
                                    colorOrder,