diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 79889f628..284cc6a84 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -180,7 +180,7 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] python-architecture: [x64, x86] fail-fast: false steps: @@ -235,7 +235,7 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] fail-fast: false steps: - name: Cache .hunter folder @@ -329,7 +329,7 @@ jobs: # echo "DEPTHAI_INSTALLATION_DIR=$PWD/build_core/install/" >> $GITHUB_ENV - name: Build wheels - run: for PYBIN in {9..11}; do "python3.${PYBIN}" -m pip wheel . -w wheelhouse/ --verbose; done + run: for PYBIN in {9..12}; do "python3.${PYBIN}" -m pip wheel . -w wheelhouse/ --verbose; done - name: Auditing wheels run: delocate-wheel -v -w wheelhouse/audited wheelhouse/*.whl @@ -352,7 +352,7 @@ jobs: needs: build-docstrings runs-on: ubuntu-latest container: - image: quay.io/pypa/manylinux2014_x86_64:2022-10-30-402504a + image: quay.io/pypa/manylinux2014_x86_64:2024-01-08-eb135ed env: PLAT: manylinux2014_x86_64 steps: @@ -395,7 +395,7 @@ jobs: /opt/python/cp38-cp38/bin/python3.8 setup.py sdist --formats=gztar mv dist/* wheelhouse/audited/ - name: Build wheels - run: for PYBIN in /opt/python/cp3{6..11}*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done + run: for PYBIN in /opt/python/cp3{6..12}*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done - name: Audit wheels run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done - name: Archive wheel artifacts @@ -416,7 +416,7 @@ jobs: needs: build-docstrings runs-on: [self-hosted, linux, ARM64] container: - image: quay.io/pypa/manylinux2014_aarch64:2022-10-30-402504a + image: quay.io/pypa/manylinux2014_aarch64:2024-01-08-eb135ed env: PLAT: manylinux2014_aarch64 # Mount local hunter cache directory, instead of transfering to Github and back @@ -452,7 +452,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/v') != true run: echo "BUILD_COMMIT_HASH=${{github.sha}}" >> $GITHUB_ENV - name: Building wheels - run: for PYBIN in /opt/python/cp3{6..11}*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done + run: for PYBIN in /opt/python/cp3{6..12}*/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done - name: Auditing wheels run: for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done - name: Archive wheel artifacts @@ -573,4 +573,4 @@ jobs: workflow_timeout_seconds: 300 # was 120 Default: 300 - name: Release - run: echo "https://github.com/luxonis/depthai-core-hil-tests/actions/runs/${{steps.return_dispatch.outputs.run_id}}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + run: echo "https://github.com/luxonis/depthai-core-hil-tests/actions/runs/${{steps.return_dispatch.outputs.run_id}}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/test-install-dependencies.yml b/.github/workflows/test-install-dependencies.yml index bc08b2c62..7300ddb8b 100644 --- a/.github/workflows/test-install-dependencies.yml +++ b/.github/workflows/test-install-dependencies.yml @@ -16,7 +16,7 @@ runs-on: ubuntu-latest strategy: matrix: - container_image: ["fedora:34", "fedora:35", "fedora:36", "ubuntu:18.04", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:22.10"] + container_image: ["fedora:34", "fedora:35", "fedora:36", "ubuntu:18.04", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:rolling"] container: image: ${{ matrix.container_image }} steps: @@ -32,13 +32,23 @@ ln -snf /usr/share/zoneinfo/UTC /etc/localtime && echo UTC > /etc/timezone # Otherwise tzdata installer prompts for user input sed '/udevadm control --reload-rules && sudo udevadm trigger/d' docs/source/_static/install_dependencies.sh > tmp_script.sh # Doesn't work on docker bash tmp_script.sh + - name: Create a virtual environment + if: endsWith(matrix.container_image, 'rolling') == true + run: | + sudo apt-get install -y python3-venv + python3 -m venv venv + . venv/bin/activate + pip install --upgrade pip + python3 examples/install_requirements.py + shell: bash - name: Install example requirements + if: endsWith(matrix.container_image, 'rolling') == false run: | python3 examples/install_requirements.py test_macos: strategy: matrix: - os: ["macos-11", "macos-12"] + os: ["macos-11", "macos-12", "macos-13", "macos-14"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -53,12 +63,16 @@ runs-on: windows-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" - name: Download chocolatey shell: pwsh run: Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - - name: Install dependencies + - name: Install pycharm-community dependency shell: pwsh - run: choco install cmake git python --version 3.10 -y - - name: Install example requrirements + run: choco install pycharm-community -y + - name: Install example requirements run: | python examples/install_requirements.py + diff --git a/docs/source/_static/images/components/device_timesync.jpg b/docs/source/_static/images/components/device_timesync.jpg index f94b3b6f8..115ef5ffb 100644 Binary files a/docs/source/_static/images/components/device_timesync.jpg and b/docs/source/_static/images/components/device_timesync.jpg differ diff --git a/docs/source/_static/images/examples/depth_video_synced.gif b/docs/source/_static/images/examples/depth_video_synced.gif new file mode 100644 index 000000000..1c37fd5e3 Binary files /dev/null and b/docs/source/_static/images/examples/depth_video_synced.gif differ diff --git a/docs/source/_static/install_dependencies.sh b/docs/source/_static/install_dependencies.sh index 9b25c7a08..7298def5f 100755 --- a/docs/source/_static/install_dependencies.sh +++ b/docs/source/_static/install_dependencies.sh @@ -12,7 +12,7 @@ readonly linux_pkgs=( python3-numpy ) -readonly ubuntu_pkgs=( +readonly debian_pkgs=( ${linux_pkgs[@]} # https://docs.opencv.org/master/d7/d9f/tutorial_linux_install.html build-essential @@ -22,7 +22,6 @@ readonly ubuntu_pkgs=( libavformat-dev libswscale-dev python3-dev - libtbb2 libtbb-dev libjpeg-dev libpng-dev @@ -31,7 +30,6 @@ readonly ubuntu_pkgs=( ffmpeg libsm6 libxext6 - libgl1-mesa-glx python3-pyqt5 python3-pyqt5.qtquick qml-module-qtquick-controls2 @@ -46,30 +44,65 @@ readonly ubuntu_pkgs=( qml-module-qtquick-window2 ) -readonly ubuntu_pkgs_pre22_04=( - "${ubuntu_pkgs[@]}" - libdc1394-22-dev -) - -readonly ubuntu_pkgs_post22_04=( - "${ubuntu_pkgs[@]}" - libdc1394-dev -) - -readonly ubuntu_arm_pkgs=( - "${ubuntu_pkgs[@]}" - libdc1394-22-dev +readonly debian_arm_pkgs=( + ${linux_pkgs[@]} + # https://docs.opencv.org/master/d7/d9f/tutorial_linux_install.html + build-essential + libgtk2.0-dev + pkg-config + libavcodec-dev + libavformat-dev + libswscale-dev + python3-dev + libtbb-dev + libjpeg-dev + libpng-dev + libtiff-dev + # https://stackoverflow.com/questions/55313610 + ffmpeg + libsm6 + libxext6 + python3-pyqt5 + python3-pyqt5.qtquick + qml-module-qtquick-controls2 + qml-module-qt-labs-platform + qtdeclarative5-dev + qml-module-qtquick2 + qtbase5-dev + qtchooser + qt5-qmake + qtbase5-dev-tools + qml-module-qtquick-layouts + qml-module-qtquick-window2 # https://stackoverflow.com/a/53402396/5494277 libhdf5-dev libhdf5-dev libatlas-base-dev - libjasper-dev # https://github.com/EdjeElectronics/TensorFlow-Object-Detection-on-the-Raspberry-Pi/issues/18#issuecomment-433953426 libilmbase-dev libopenexr-dev libgstreamer1.0-dev ) +readonly debian_pkgs_pre22_04=( + libdc1394-22-dev + libgl1-mesa-glx + libtbb2 + +) +readonly debian_pkgs_post22_04=( + libdc1394-dev + libgl1-mesa-glx + libtbbmalloc2 + +) +readonly debian_pkgs_23=( + libdc1394-dev + libgl1-mesa-dev + libtbbmalloc2 +) + + readonly fedora_pkgs=( ${linux_pkgs[@]} gtk2-devel @@ -94,6 +127,33 @@ print_and_exec () { $* } +version_lte() { + [[ "$1" == "$(echo -e "$1\n$2" | sort -V | head -n1)" ]] +} + +declare -A debian_versions=( + ["trixie/sid"]="13" + ["bookworm/sid"]="12" + ["bullseye/sid"]="11" + ["buster/sid"]="10" + ["stretch/sid"]="9" + ["jessie/sid"]="8" + ["wheezy/sid"]="7" + ["squeeze/sid"]="6" +) + +# Function to lookup and print Debian version number +lookup_debian_version_number() { + debian_version_string="$1" + version_number="${debian_versions[$debian_version_string]}" + + if [ -n "$version_number" ]; then + echo "$version_number" + else + echo "None" + fi +} + if [[ $(uname) == "Darwin" ]]; then echo "During Homebrew install, certain commands need 'sudo'. Requesting access..." sudo true @@ -105,46 +165,87 @@ if [[ $(uname) == "Darwin" ]]; then echo echo "=== Installed successfully! IMPORTANT: For changes to take effect," echo "please close and reopen the terminal window, or run: exec \$SHELL" + elif [ -f /etc/os-release ]; then - # shellcheck source=/etc/os-release source /etc/os-release + if [ -f /etc/debian_version ]; then + output=$(cat /etc/debian_version) + echo $output + if [[ $output == *sid ]]; then + version=$(lookup_debian_version_number $output) + else + version=$output + fi + + # Correctly determine if the architecture is ARM or aarch64 + IS_ARM=false + if [[ $(uname -m) =~ ^arm* || $(uname -m) == "aarch64" ]]; then + IS_ARM=true + fi + + echo "$version" + echo "$IS_ARM" + + if [ $IS_ARM ]; then + sudo DEBIAN_FRONTEND=noninteractive apt install -y "${debian_arm_pkgs[@]}" + if [[ $version == 13* ]]; then + echo "Detected ARM Debian 13" + sudo apt install -y "${debian_pkgs_23[@]}" + elif version_lte "$version" "11.99"; then + echo "Using pre-22.04 ARM package list" + sudo apt-get install -y ${debian_pkgs_pre22_04[@]} + + # Check for uvcdynctrl package and recommend removal if found + if dpkg -s uvcdynctrl &> /dev/null; then + echo -e "\033[33mWe detected 'uvcdynctrl' installed on your system.\033[0m" + # Instructions for removal + echo -e "\033[33m$ sudo apt remove uvcdynctrl uvcdynctrl-data\033[0m" + echo -e "\033[33m$ sudo rm -f /var/log/uvcdynctrl-udev.log\033[0m" + fi + - if [[ "$ID" == "ubuntu" || "$ID" == "debian" || "$ID_LIKE" == "ubuntu" || "$ID_LIKE" == "debian" || "$ID_LIKE" == "ubuntu debian" ]]; then - if [[ ! $(uname -m) =~ ^arm* ]]; then - sudo apt-get update - if [[ "$VERSION_ID" > "22.04" || "$VERSION_ID" == "22.04" ]]; then - sudo apt-get install -y "${ubuntu_pkgs_post22_04[@]}" else - sudo apt-get install -y "${ubuntu_pkgs_pre22_04[@]}" + echo "Using post-22.04 ARM package list" + sudo apt-get install -y ${debian_pkgs_post22_04[@]} + fi + + # Add libjasper-dev for ARM but not aarch64 + [[ $(uname -m) =~ ^arm* ]] && { sudo apt install -y libjasper-dev; } + + else + sudo DEBIAN_FRONTEND=noninteractive apt install -y "${debian_pkgs[@]}" + if [[ $version == 13* ]]; then + echo "Detected Debian 13" + sudo apt install -y "${debian_pkgs_23[@]}" + elif version_lte "$version" "11.99"; then + echo "Using pre-22.04 package list" + sudo apt-get install -y "${debian_pkgs_pre22_04[@]}" + + else + echo "Using post-22.04 package list" + sudo apt-get install -y "${debian_pkgs_post22_04[@]}" fi - python3 -m pip install --upgrade pip - elif [[ $(uname -m) =~ ^arm* ]]; then - sudo apt-get update - sudo apt-get install -y "${ubuntu_arm_pkgs[@]}" - python3 -m pip install --upgrade pip fi - # As set -e is set, retrieve the return value without exiting - RET=0 - dpkg -s uvcdynctrl > /dev/null 2>&1 || RET=$? || true - # is uvcdynctrl installed - if [[ "$RET" == "0" ]]; then - echo -e "\033[33mWe detected \"uvcdynctrl\" installed on your system. \033[0m" - echo -e "\033[33mWe recommend removing this package, as it creates a huge log files if a camera is used in UVC mode (webcam)\033[0m" - echo -e "\033[33mYou can do so by running the following commands:\033[0m" - echo -e "\033[33m$ sudo apt remove uvcdynctrl uvcdynctrl-data\033[0m" - echo -e "\033[33m$ sudo rm -f /var/log/uvcdynctrl-udev.log\033[0m" - echo "" + # Check for uvcdynctrl package and recommend removal if found + if dpkg -s uvcdynctrl &> /dev/null; then + echo -e "\033[33mWe detected 'uvcdynctrl' installed on your system.\033[0m" + # Instructions for removal + echo -e "\033[33m$ sudo apt remove uvcdynctrl uvcdynctrl-data\033[0m" + echo -e "\033[33m$ sudo rm -f /var/log/uvcdynctrl-udev.log\033[0m" fi - OS_VERSION=$(lsb_release -r |cut -f2) - if [ "$OS_VERSION" == "21.04" ]; then + + + if [ "$VERSION_ID" == "21.04" ]; then echo -e "\033[33mThere are known issues with running our demo script on Ubuntu 21.04, due to package \"python3-pyqt5.sip\" not being in a correct version (>=12.9)\033[0m" echo -e "\033[33mWe recommend installing the updated version manually using the following commands\033[0m" echo -e "\033[33m$ wget http://mirrors.kernel.org/ubuntu/pool/universe/p/pyqt5-sip/python3-pyqt5.sip_12.9.0-1_amd64.deb\033[0m" echo -e "\033[33m$ sudo dpkg -i python3-pyqt5.sip_12.9.0-1_amd64.deb\033[0m" echo "" fi + + elif [[ "$ID" == "fedora" ]]; then sudo dnf update -y sudo dnf install -y "${fedora_pkgs[@]}" @@ -154,7 +255,6 @@ elif [ -f /etc/os-release ]; then echo "ERROR: Distribution not supported" exit 99 fi - # Allow all users to read and write to Myriad X devices echo "Installing udev rules..." echo 'SUBSYSTEM=="usb", ATTRS{idVendor}=="03e7", MODE="0666"' | sudo tee /etc/udev/rules.d/80-movidius.rules > /dev/null diff --git a/docs/source/components/device.rst b/docs/source/components/device.rst index 4fe7466a2..0f2e74fc8 100644 --- a/docs/source/components/device.rst +++ b/docs/source/components/device.rst @@ -56,7 +56,7 @@ subnet, you can specify the device (either with MxID, IP, or USB port name) you # ... Host clock syncing -================== +################## When depthai library connects to a device, it automatically syncs device's timestamp to host's timestamp. Timestamp syncing happens continuously at around 5 second intervals, and can be configured via API (example script below). diff --git a/docs/source/components/messages/message_group.rst b/docs/source/components/messages/message_group.rst new file mode 100644 index 000000000..e947ad67d --- /dev/null +++ b/docs/source/components/messages/message_group.rst @@ -0,0 +1,32 @@ +MessageGroup +============ + +The MessageGroup message type is a versatile container used in DepthAI pipelines to group together a map of arbitrary DepthAI messages. It serves as the primary output of the :ref:`Sync` node, effectively synchronizing various input streams, and acts as the input to the :ref:`MessageDemux` node for subsequent disaggregation and processing. + +Creating MessageGroup +##################### + +MessageGroup can be created automatically by the Sync node as it aligns and groups messages from different sources based on their timestamps. Alternatively, it can be manually constructed in a host application or within a :ref:`Script` node to create custom groupings of DepthAI messages. + +Reference +######### + +.. tabs:: + + .. tab:: Python + + .. autoclass:: depthai.MessageGroup + :members: + :inherited-members: + :noindex: + + .. tab:: C++ + + .. doxygenclass:: dai::MessageGroup + :project: depthai-core + :members: + :private-members: + :undoc-members: + + +.. include:: ../../includes/footer-short.rst diff --git a/docs/source/components/nodes/camera.rst b/docs/source/components/nodes/camera.rst index 32740baa8..9f9bfb412 100644 --- a/docs/source/components/nodes/camera.rst +++ b/docs/source/components/nodes/camera.rst @@ -9,6 +9,7 @@ Compared to :ref:`ColorCamera` node, Camera node: - Supports **cam.setSize()**, which replaces both ``cam.setResolution()`` and ``cam.setIspScale()``. Camera node will automatically find resolution that fits best, and apply correct scaling to achieve user-selected size - Supports **cam.setCalibrationAlpha()**, example here: :ref:`Undistort camera stream` - Supports **cam.loadMeshData()** and **cam.setMeshStep()**, which can be used for custom image warping (undistortion, perspective correction, etc.) +- Automatically undistorts camera stream if HFOV of the camera is greater than 85°. You can disable this with: ``cam.setMeshSource(dai.CameraProperties.WarpMeshSource.NONE)``. Besides points above, compared to :ref:`MonoCamera` node, Camera node: diff --git a/docs/source/components/nodes/message_demux.rst b/docs/source/components/nodes/message_demux.rst new file mode 100644 index 000000000..bb6d3ee85 --- /dev/null +++ b/docs/source/components/nodes/message_demux.rst @@ -0,0 +1,122 @@ +MessageDemux +============ + + +The MessageDemux (Demultiplexer) node is used to separate a :ref:`MessageGroup` into individual outputs. It currently serves as way to demultiplex the output of :ref:`Sync` node. + +How to Place It +############### + +.. tabs:: + + .. code-tab:: py + + pipeline = dai.Pipeline() + demux = pipeline.create(dai.node.MessageDemux) + + .. code-tab:: c++ + + dai::Pipeline pipeline; + auto demux = pipeline.create(); + + +Inputs and Outputs +################## + +.. code-block:: + + ┌───────────────────┐ + input │ │ + ──────────────►│ │ + │ MessageDemux │ output1 + │ ├───────────► + │ │ output2 + │ ├───────────► + │ │ ... + └───────────────────┘ + +**Message types** + +- :code:`input` - :ref:`MessageGroup` +- :code:`output1`, :code:`output2`, ... - Individual output messages + +Usage +##### + +The MessageDemux node is particularly useful for handling different types of data coming from a single source. +For example, when the :ref:`Sync` node is used to synchronize the outputs of multiple nodes, the output of the Sync node is a :ref:`MessageGroup` containing all the messages from the synchronized nodes. The Demux node can be used to separate the messages into individual streams. + +.. tabs:: + + .. code-tab:: py + + # Create sync node and set sync threshold + sync = pipeline.create(dai.node.Sync) + sync.setSyncThresholdMs(timedelta(milliseconds=100)) + + # Create demux node + demux = pipeline.create(dai.node.MessageDemux) + + # Sync the outputs of multiple nodes + rgb.preview.link(sync.inputs["rgb"]) + stereo.depth.link(sync.inputs["depth"]) + script.outputs["out"].link(sync.inputs["script"]) + + sync.out.link(demux.input) # Sync output is a MessageGroup containing all the messages from the synchronized nodes + + # Demux the MessageGroup into individual messages + demux.outputs["rgb"].link(xout1.input) + demux.outputs["depth"].link(xout2.input) + demux.outputs["script"].link(xout3.input) + + + + + .. code-tab:: c++ + + // Create sync node and set sync threshold + auto sync = pipeline.create(); + sync->setSyncThreshold(std::chrono::milliseconds(100)); + + // Create demux node + auto demux = pipeline.create(); + + // Sync the outputs of multiple nodes + rgb.preview.link(sync->input["rgb"]); + stereo.depth.link(sync->input["depth"]); + script.outputs["out"].link(sync->input["script"]); + + sync->out.link(demux->input); // Sync output is a MessageGroup containing all the messages from the synchronized nodes + + // Demux the MessageGroup into individual messages + demux->outputs["rgb"].link(xout1.input); + demux->outputs["depth"].link(xout2.input); + demux->outputs["script"].link(xout3.input); + + +Examples of Functionality +########################## + +- :ref:`Demuxing Synchronized Script Outputs` + +Reference +######### + +.. tabs:: + + .. tab:: Python + + .. autoclass:: depthai.node.MessageDemux + :members: + :inherited-members: + :noindex: + + .. tab:: C++ + + .. doxygenclass:: dai::node::MessageDemux + :project: depthai-core + :members: + :private-members: + :undoc-members: + +.. include:: ../../includes/footer-short.rst diff --git a/docs/source/components/nodes/sync_node.rst b/docs/source/components/nodes/sync_node.rst new file mode 100644 index 000000000..913081890 --- /dev/null +++ b/docs/source/components/nodes/sync_node.rst @@ -0,0 +1,115 @@ +Sync +==== + +The Sync node is used for synchronizing multiple input streams based on their timestamps. It outputs a grouped message containing synchronized frames from the input streams. +The output message is a :ref:`MessageGroup` containing synchronized messages from all the input streams. These can be demultiplexed using the :ref:`MessageDemux` node. + +How to Place it +############### + +.. tabs:: + + .. code-tab:: py + + pipeline = dai.Pipeline() + sync = pipeline.create(dai.node.Sync) + + .. code-tab:: c++ + + dai::Pipeline pipeline; + auto sync = pipeline.create(); + + +Inputs and Outputs +################## + +.. code-block:: + + ┌───────────────────┐ + input1 │ │ + ──────────────►│ │ + input2 │ │ out + ──────────────►│ Sync ├───────────► + │ │ + ... │ │ + ──────────────►│ │ + └───────────────────┘ + +**Message types** + +- :code:`input1`, :code:`input2`, ... - any message type from :ref:`Messages` +- :code:`out` - :ref:`MessageGroup` + + +Message Synchronization +######################## + +The Sync node aligns incoming messages based on their timestamps. The synchronization criteria and behavior can be configured using the :code:`depthai.node.Sync.setSyncThreshold` and :code:`depthai.node.Sync.setSyncAttempts` method. More info in the :ref:`API Reference `. + + +Usage +##### + +.. tabs:: + + .. code-tab:: py + + pipeline = dai.Pipeline() + sync = pipeline.create(dai.node.Sync) + + # Configure threshold for timestamp alignment + sync.setSyncThreshold(timedelta(milliseconds=50)) + + # Configure inputs to be synchronized + camRgb.video.link(sync.inputs["input1"]) + stereo.depth.link(sync.inputs["input2"]) + + sync.output.link(xout.input) + # ... + + .. code-tab:: c++ + + dai::Pipeline pipeline; + auto sync = pipeline.create(); + + // Configure threshold for timestamp alignment + sync->setSyncThreshold(std::chrono::milliseconds(50)); + + // Configure inputs to be synchronized + camRgb.video.link(sync->input["input1"]); + stereo.depth.link(sync->input["input2"]); + + sync->out.link(xout.input); + // ... + + + +Examples of Functionality +########################## + +- :ref:`Depth and Video Sync` +- :ref:`Multiple Scripts Sync` +- :ref:`IMU and Video Sync` +- :ref:`Demuxing Synchronized Script Outputs` + +Reference +######### + +.. tabs:: + + .. tab:: Python + + .. autoclass:: depthai.node.Sync + :members: + :inherited-members: + :noindex: + + .. tab:: C++ + + .. doxygenclass:: dai::node::Sync + :project: depthai-core + :members: + :private-members: + :undoc-members: + +.. include:: ../../includes/footer-short.rst diff --git a/docs/source/components/nodes/video_encoder.rst b/docs/source/components/nodes/video_encoder.rst index 2c3224219..522c74962 100644 --- a/docs/source/components/nodes/video_encoder.rst +++ b/docs/source/components/nodes/video_encoder.rst @@ -53,10 +53,13 @@ Usage # Set H265 encoding for the ColorCamera video output videoEncoder = pipeline.create(dai.node.VideoEncoder) videoEncoder.setDefaultProfilePreset(cam.getFps(), dai.VideoEncoderProperties.Profile.H265_MAIN) + videoEncoder.setBitrateKbps(500) # 0.5 Mbps # Create MJPEG encoding for still images stillEncoder = pipeline.create(dai.node.VideoEncoder) stillEncoder.setDefaultProfilePreset(1, dai.VideoEncoderProperties.Profile.MJPEG) + # stillEncoder.setLossless(True) # Lossless only for MJPEG + stillEncoder.setQuality(90) # 0-100, 100 being the best quality (not lossless though) cam.still.link(stillEncoder.input) cam.video.link(videoEncoder.input) @@ -69,10 +72,13 @@ Usage // Set H265 encoding for the ColorCamera video output auto videoEncoder = pipeline.create(); videoEncoder->setDefaultProfilePreset(cam->getFps(), dai::VideoEncoderProperties::Profile::H265_MAIN); + videoEncoder->setBitrateKbps(500); // 0.5 Mbps // Create MJPEG encoding for still images stillEncoder = pipeline.create(dai.node.VideoEncoder); stillEncoder->setDefaultProfilePreset(1, dai::VideoEncoderProperties::Profile::MJPEG); + // stillEncoder->setLossless(true); // Lossless only for MJPEG + stillEncoder->setQuality(90); // 0-100, 100 being the best quality (not lossless though) cam->still.link(stillEncoder->input); cam->video.link(videoEncoder->input); diff --git a/docs/source/samples/StereoDepth/stereo_depth_video.rst b/docs/source/samples/StereoDepth/stereo_depth_video.rst index 7b2d7e496..3ea27e03a 100644 --- a/docs/source/samples/StereoDepth/stereo_depth_video.rst +++ b/docs/source/samples/StereoDepth/stereo_depth_video.rst @@ -11,6 +11,16 @@ inside the code: #. `extended`: suitable for short range objects. For more information `click here `__ #. `subpixel`: suitable for long range. For more information `click here `__ +Stereo Alpha Param +################## + +With `--alpha` you can select scaling during the rectification. Values between 0 and 1: + +- 0.0 (default value) means that the rectified images are zoomed and shifted so that only valid pixels are visible (no black areas after rectification) +- 1.0 means that the rectified image is decimated and shifted so that all the pixels from the original images from the cameras are retained in the rectified images (no source image pixels are lost). + +.. image:: https://github.com/luxonis/depthai-python/assets/18037362/887d489a-6bf6-41ac-9b8f-06ab7b0cd488 + .. rubric:: Similar samples: - :ref:`Depth Preview` diff --git a/docs/source/samples/Sync/demux_message_group.rst b/docs/source/samples/Sync/demux_message_group.rst new file mode 100644 index 000000000..5cbd08be8 --- /dev/null +++ b/docs/source/samples/Sync/demux_message_group.rst @@ -0,0 +1,78 @@ +Demuxing Synchronized Script Outputs +==================================== + +This example demonstrates the use of the DepthAI Sync node in conjunction with the Demux node to synchronize and then demux outputs from two separate script nodes. Each script node generates data buffers at different intervals, which are first synchronized by the Sync node and then demultiplexed by the MessageDemux node. + +.. rubric:: Similar samples: + +- :ref:`Multiple Scripts Sync` +- :ref:`Depth and Video Sync` +- :ref:`IMU and Video Sync` + +Demo +#### + + +.. code-block:: + + ~/depthai-python/examples/Sync $ python3 demux_message_group.py + Start + Buffer 1 timestamp: 0:00:03.581073 + Buffer 2 timestamp: 0:00:03.591084 + ---------- + Buffer 1 timestamp: 0:00:04.583100 + Buffer 2 timestamp: 0:00:04.497079 + ---------- + Buffer 1 timestamp: 0:00:06.587174 + Buffer 2 timestamp: 0:00:06.611154 + ---------- + Buffer 1 timestamp: 0:00:07.589147 + Buffer 2 timestamp: 0:00:07.517125 + ---------- + Buffer 1 timestamp: 0:00:09.593076 + Buffer 2 timestamp: 0:00:09.631089 + ---------- + Buffer 1 timestamp: 0:00:10.595106 + Buffer 2 timestamp: 0:00:10.537082 + + +Setup +##### + +.. include:: /includes/install_from_pypi.rst + + +Source code +########### + +.. tabs:: + + .. tab:: Python + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../examples/Sync/demux_message_group.py + :language: python + :linenos: + + .. tab:: C++ + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../depthai-core/examples/Sync/demux_message_group.cpp + :language: cpp + :linenos: + +How it Works +############ + +#. Initialize a DepthAI pipeline. +#. Create two Script nodes, with each script generating and sending data buffers at different intervals. +#. Set up a Sync node with a synchronization threshold. +#. Integrate a MessageDemux node to separate the synchronized data streams. +#. Link the outputs of the Script nodes to the Sync node, and then from the Sync node to the MessageDemux node. +#. Start the pipeline and continuously receive demultiplexed data from the MessageDemux node. +#. Print the timestamps of the demultiplexed data for comparison. + + +.. include:: /includes/footer-short.rst diff --git a/docs/source/samples/Sync/depth_video_sync.rst b/docs/source/samples/Sync/depth_video_sync.rst new file mode 100644 index 000000000..a9b62c4e3 --- /dev/null +++ b/docs/source/samples/Sync/depth_video_sync.rst @@ -0,0 +1,59 @@ +Depth and Video Sync +==================== + +This example demonstrates the use of the DepthAI Sync node to synchronize output from StereoDepth and Color Camera nodes. It showcases how to process and display disparity maps from stereo cameras and video frames from a color camera in real time. + + +.. rubric:: Similar samples: + +- :ref:`IMU and Video Sync` +- :ref:`Multiple Scripts Sync` + + +Demo +#### + +.. image:: ../../../../docs/source/_static/images/examples/depth_video_synced.gif + :alt: Depth and Video Sync Demo + :width: 100% + :align: center + + +Setup +##### + +.. include:: /includes/install_from_pypi.rst + + +Source code +########### + +.. tabs:: + + .. tab:: Python + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../examples/Sync/depth_video_synced.py + :language: python + :linenos: + + .. tab:: C++ + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../depthai-core/examples/Sync/depth_video_synced.cpp + :language: cpp + :linenos: + +How it Works +############ + +#. Initialize MonoCamera nodes for left and right cameras. +#. Set up a ColorCamera node. +#. Create a StereoDepth node for depth perception. +#. Configure the Sync node to synchronize disparity from the StereoDepth node and video frames from the ColorCamera node. +#. Display the synchronized frames using OpenCV. Frames are synchronized to threshold of 50 milliseconds. + + +.. include:: /includes/footer-short.rst diff --git a/docs/source/samples/Sync/imu_video_sync.rst b/docs/source/samples/Sync/imu_video_sync.rst new file mode 100644 index 000000000..ab2dbb1b9 --- /dev/null +++ b/docs/source/samples/Sync/imu_video_sync.rst @@ -0,0 +1,82 @@ +IMU and Video Sync +================== + +This example demonstrates the use of the DepthAI Sync node to synchronize IMU (Inertial Measurement Unit) data with video frames from a color camera. It highlights the capability to process and display the latest rotation vector from the IMU alongside the video stream in real-time. + +.. rubric:: Similar samples: + +- :ref:`Depth and Video Sync` +- :ref:`Multiple Scripts Sync` + +Demo +#### + +.. code-block:: + + ~/depthai-python/examples/Sync $ python3 imu_video_synced.py + IMU type: BNO086, firmware version: 3.9.7 + + Device timestamp imu: 0:00:05.379914 + Device timestamp video:0:00:05.385096 + Quaternion: i: -0.0549 j: -0.0335 k: 0.0018 real: 0.9979 + + + Device timestamp imu: 0:00:05.410274 + Device timestamp video:0:00:05.418425 + Quaternion: i: -0.0549 j: -0.0334 k: 0.0018 real: 0.9979 + + + Device timestamp imu: 0:00:05.445439 + Device timestamp video:0:00:05.451753 + Quaternion: i: -0.0548 j: -0.0334 k: 0.0018 real: 0.9979 + + + Device timestamp imu: 0:00:05.475084 + Device timestamp video:0:00:05.485082 + Quaternion: i: -0.0547 j: -0.0334 k: 0.0018 real: 0.9979 + + + Device timestamp imu: 0:00:05.510046 + Device timestamp video:0:00:05.518411 + Quaternion: i: -0.0546 j: -0.0334 k: 0.0018 real: 0.9979 + + +Setup +##### + +.. include:: /includes/install_from_pypi.rst + + +Source code +########### + +.. tabs:: + + .. tab:: Python + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../examples/Sync/imu_video_synced.py + :language: python + :linenos: + + .. tab:: C++ + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../depthai-core/examples/Sync/imu_video_synced.cpp + :language: cpp + :linenos: + +How it Works +############ + +#. Initialize the DepthAI device. +#. Check the connected IMU type and firmware version. +#. Create a pipeline and add a ColorCamera and IMU node. +#. Set up the Sync node to synchronize the IMU data with the video frames. +#. Link the output of the ColorCamera and IMU nodes to the Sync node. +#. Start the pipeline and continuously receive synchronized data. +#. Display the video frames and print the IMU rotation vector data, including quaternion values. + +.. include:: /includes/footer-short.rst diff --git a/docs/source/samples/Sync/scripts_sync.rst b/docs/source/samples/Sync/scripts_sync.rst new file mode 100644 index 000000000..73236d48b --- /dev/null +++ b/docs/source/samples/Sync/scripts_sync.rst @@ -0,0 +1,74 @@ +Multiple Scripts Sync +===================== + +This example illustrates the use of the DepthAI Sync node to synchronize outputs from two separate script nodes. Each script generates and sends data buffers at different intervals, and the Sync node aligns these outputs based on their timestamps. + +.. rubric:: Similar samples: + +- :ref:`Depth and Video Sync` +- :ref:`IMU and Video Sync` + +Demo +#### + +.. code-block:: + + ~/depthai-python/examples/Sync $ python3 sync_scripts.py + Start + Received s1 with timestamp 0:00:02.420089 + Received s2 with timestamp 0:00:02.461076 + Time interval between messages: 40.987ms + ---------- + Received s1 with timestamp 0:00:03.422108 + Received s2 with timestamp 0:00:03.367069 + Time interval between messages: 55.039ms + ---------- + Received s1 with timestamp 0:00:05.426088 + Received s2 with timestamp 0:00:05.481086 + Time interval between messages: 54.998ms + ---------- + Received s1 with timestamp 0:00:06.428106 + Received s2 with timestamp 0:00:06.387129 + Time interval between messages: 40.977ms + ---------- + +Setup +##### + +.. include:: /includes/install_from_pypi.rst + + +Source code +########### + +.. tabs:: + + .. tab:: Python + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../examples/Sync/sync_scripts.py + :language: python + :linenos: + + .. tab:: C++ + + Also `available on GitHub `__ + + .. literalinclude:: ../../../../depthai-core/examples/Sync/sync_scripts.cpp + :language: cpp + :linenos: + + +How it Works +############ + +#. Initialize a DepthAI pipeline. +#. Create two Script nodes, each generating and sending data buffers at different intervals. +#. Set up a Sync node with a synchronization threshold. +#. Link the outputs of the Script nodes to the Sync node. +#. Start the pipeline and continuously receive synchronized data from the Script nodes. +#. Print the received data along with timestamps and the interval between messages. + + +.. include:: /includes/footer-short.rst diff --git a/docs/source/samples/calibration/calibration_reader.rst b/docs/source/samples/calibration/calibration_reader.rst index 247d77254..f9d928730 100644 --- a/docs/source/samples/calibration/calibration_reader.rst +++ b/docs/source/samples/calibration/calibration_reader.rst @@ -27,7 +27,7 @@ Here's theoretical calculation of the focal length in pixels: .. math:: - f_x = \text{width}_x \cdot \frac{1}{\tan \left(\frac{\text{HFOV}_x}{2}\frac{\pi}{180}\right)} + f_x = \text{width}_x \cdot \frac{1}{2 \cdot \tan \left(\frac{\text{HFOV}_x}{2}\frac{\pi}{180}\right)} To get the HFOV you can use `this script `__, which also works for wide-FOV cameras and allows you to specif alpha parameter. @@ -36,13 +36,13 @@ With 400P (640x400) camera resolution where HFOV=71.9 degrees: .. math:: - f_x = 640 \cdot \frac{1}{\tan \left(\frac{71.9}{2}\frac{\pi}{180}\right)} = 441.25 + f_x = 640 \cdot \frac{1}{2 \cdot \tan \left(\frac{71.9}{2}\frac{\pi}{180}\right)} = 441.25 And for 800P (1280x800) camera resolution where HFOV=71.9 degrees: .. math:: - f_x = 1280 \cdot \frac{1}{\tan \left(\frac{71.9}{2}\frac{\pi}{180}\right)} = 882.5 + f_x = 1280 \cdot \frac{1}{2 \cdot \tan \left(\frac{71.9}{2}\frac{\pi}{180}\right)} = 882.5 diff --git a/docs/source/tutorials/code_samples.rst b/docs/source/tutorials/code_samples.rst index eed30756a..cbb25f600 100644 --- a/docs/source/tutorials/code_samples.rst +++ b/docs/source/tutorials/code_samples.rst @@ -23,6 +23,7 @@ Code Samples ../samples/Script/* ../samples/SpatialDetection/* ../samples/StereoDepth/* + ../samples/Sync/* ../samples/SystemLogger/* ../samples/ToF/* ../samples/VideoEncoder/* @@ -159,6 +160,15 @@ are presented with code. - :ref:`Stereo Depth Video` - An extended version of **Depth Preview** - :ref:`RGB Depth alignment` - Displays RGB depth aligned frames + +.. rubric:: Sync Node + +- :ref:`Depth and Video Sync` - Synchronizes depth and video streams for real-time processing and display. +- :ref:`IMU and Video Sync` - Aligns IMU data with video frames, showcasing real-time rotation vector display alongside video. +- :ref:`Multiple Scripts Sync` - Demonstrates synchronization of data from two script nodes emitting data at different intervals. +- :ref:`Demuxing Synchronized Script Outputs` - Features synchronization of script outputs followed by demultiplexing for distinct processing streams. + + .. rubric:: SystemLogger - :ref:`System information` - Displays device system information (memory/cpu usage, temperature) diff --git a/docs/source/tutorials/message_syncing.rst b/docs/source/tutorials/message_syncing.rst index bf8dceefa..da03b0fd1 100644 --- a/docs/source/tutorials/message_syncing.rst +++ b/docs/source/tutorials/message_syncing.rst @@ -67,6 +67,11 @@ As opposed to sequence number syncing, **timestamp syncing** can sync: - **IMU** results with other messages - messages with **other devices connected to the computer**, as timestamps are :ref:`synced to the host computer clock ` + +.. note:: + DepthAI 2.24 introduces **Sync node** which can be used to sync messages from different streams, or messages from different sensors (eg. IMU and color frames). See :ref:`Sync node ` for more details. + The sync node does not currently support multiple device syncing, so if you want to sync messages from multiple devices, you should use the manual approach. + Feel free to check the `demo here `__ which uses timestamps to sync IMU, color and disparity frames together, with all of these streams producing messages at different FPS. diff --git a/docs/source/tutorials/standalone_mode.rst b/docs/source/tutorials/standalone_mode.rst index df90a7470..680137b13 100644 --- a/docs/source/tutorials/standalone_mode.rst +++ b/docs/source/tutorials/standalone_mode.rst @@ -27,6 +27,9 @@ To "communicate" with the outside world (eg. a server), POE cameras can use :ref - `HTTP client `__ - `MQTT client `__ +.. note:: + Standalone mode is missing a DNS resolver, so you will need to use IP addresses instead of domain names. + Converting a demo to standalone mode #################################### diff --git a/examples/StereoDepth/depth_preview_sr.py b/examples/StereoDepth/depth_preview_sr.py new file mode 100755 index 000000000..6d82d6317 --- /dev/null +++ b/examples/StereoDepth/depth_preview_sr.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +import cv2 +import depthai as dai +import numpy as np + +# Closer-in minimum depth, disparity range is doubled (from 95 to 190): +extendedDisparity = False +# Better accuracy for longer distance, fractional disparity 32-levels: +subpixel = True +# Better handling for occlusions: +lrCheck = True + +enableRectified = True + +# Create pipeline +pipeline = dai.Pipeline() + +# Define sources and outputs +left = pipeline.create(dai.node.ColorCamera) +right = pipeline.create(dai.node.ColorCamera) + +# Create stereo +stereo = pipeline.create(dai.node.StereoDepth) +xoutDepth = pipeline.create(dai.node.XLinkOut) +xoutDepth.setStreamName("disparity") + +# Properties +left.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) +left.setCamera("left") + +right.setResolution(dai.ColorCameraProperties.SensorResolution.THE_800_P) +right.setCamera("right") + +stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY) +stereo.initialConfig.setMedianFilter(dai.MedianFilter.KERNEL_7x7) +stereo.setLeftRightCheck(lrCheck) +stereo.setExtendedDisparity(extendedDisparity) +stereo.setSubpixel(subpixel) + + +# Linking +left.isp.link(stereo.left) +right.isp.link(stereo.right) +if enableRectified: + xoutRectR = pipeline.create(dai.node.XLinkOut) + xoutRectL= pipeline.create(dai.node.XLinkOut) + xoutRectR.setStreamName("rectifiedRight") + xoutRectL.setStreamName("rectifiedLeft") + stereo.rectifiedLeft.link(xoutRectL.input) + stereo.rectifiedRight.link(xoutRectR.input) +stereo.disparity.link(xoutDepth.input) + +maxDisp = stereo.initialConfig.getMaxDisparity() + +# Connect to device and start pipeline +with dai.Device(pipeline) as device: + while not device.isClosed(): + queueNames = device.getQueueEvents() + for q in queueNames: + message = device.getOutputQueue(q).get() + # Display arrived frames + if type(message) == dai.ImgFrame: + frame = message.getCvFrame() + if 'disparity' in q: + disp = (frame * (255.0 / maxDisp)).astype(np.uint8) + disp = cv2.applyColorMap(disp, cv2.COLORMAP_JET) + cv2.imshow(q, disp) + else: + cv2.imshow(q, frame) + if cv2.waitKey(1) == ord('q'): + break + diff --git a/examples/Sync/demux_message_group.py b/examples/Sync/demux_message_group.py index 41c488f9c..2893e1fd2 100644 --- a/examples/Sync/demux_message_group.py +++ b/examples/Sync/demux_message_group.py @@ -29,7 +29,7 @@ """) sync = pipeline.create(dai.node.Sync) -sync.setSyncThresholdMs(timedelta(milliseconds=100)) +sync.setSyncThreshold(timedelta(milliseconds=100)) demux = pipeline.create(dai.node.MessageDemux) diff --git a/setup.py b/setup.py index 7dfd75015..2860af5ba 100644 --- a/setup.py +++ b/setup.py @@ -228,6 +228,7 @@ def build_extension(self, ext): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: C++", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering",