diff --git a/CMakeLists.txt b/CMakeLists.txt index 527ad83b8..beb6844b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,8 @@ set(LIBDATACHANNEL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtppacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/nalunit.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/h264packetizationhandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/av1rtppacketizer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/av1packetizationhandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/mediachainablehandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/mediahandlerelement.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/mediahandlerrootelement.cpp @@ -114,6 +116,8 @@ set(LIBDATACHANNEL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtppacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/nalunit.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264packetizationhandler.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1rtppacketizer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/av1packetizationhandler.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediachainablehandler.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediahandlerelement.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/mediahandlerrootelement.hpp diff --git a/include/rtc/av1packetizationhandler.hpp b/include/rtc/av1packetizationhandler.hpp new file mode 100644 index 000000000..ed7941de6 --- /dev/null +++ b/include/rtc/av1packetizationhandler.hpp @@ -0,0 +1,32 @@ +/** + * Copyright (c) 2023 Paul-Louis Ageneau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#ifndef RTC_AV1_PACKETIZATION_HANDLER_H +#define RTC_AV1_PACKETIZATION_HANDLER_H + +#if RTC_ENABLE_MEDIA + +#include "av1rtppacketizer.hpp" +#include "mediachainablehandler.hpp" +#include "nalunit.hpp" + +namespace rtc { + +/// Handler for AV1 packetization +class RTC_CPP_EXPORT AV1PacketizationHandler final : public MediaChainableHandler { +public: + /// Construct handler for AV1 packetization. + /// @param packetizer RTP packetizer for AV1 + AV1PacketizationHandler(shared_ptr packetizer); +}; + +} // namespace rtc + +#endif /* RTC_ENABLE_MEDIA */ + +#endif /* RTC_AV1_PACKETIZATION_HANDLER_H */ diff --git a/include/rtc/av1rtppacketizer.hpp b/include/rtc/av1rtppacketizer.hpp new file mode 100644 index 000000000..9e7432a0f --- /dev/null +++ b/include/rtc/av1rtppacketizer.hpp @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2023 Paul-Louis Ageneau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#ifndef RTC_AV1_RTP_PACKETIZER_H +#define RTC_AV1_RTP_PACKETIZER_H + +#if RTC_ENABLE_MEDIA + +#include "mediahandlerrootelement.hpp" +#include "nalunit.hpp" +#include "rtppacketizer.hpp" + +namespace rtc { + +/// RTP packetization of AV1 payload +class RTC_CPP_EXPORT AV1RtpPacketizer final : public RtpPacketizer, public MediaHandlerRootElement { + shared_ptr splitMessage(binary_ptr message); + const uint16_t maximumFragmentSize; + +public: + /// Default clock rate for AV1 in RTP + inline static const uint32_t defaultClockRate = 90 * 1000; + + // Define how OBUs are seperated in a AV1 Sample + enum class Packetization { + Obu = RTC_OBU_PACKETIZED_OBU, + TemporalUnit = RTC_OBU_PACKETIZED_TEMPORAL_UNIT, + }; + + /// Constructs AV1 payload packetizer with given RTP configuration. + /// @note RTP configuration is used in packetization process which may change some configuration + /// properties such as sequence number. + /// @param rtpConfig RTP configuration + AV1RtpPacketizer(Packetization packetization, shared_ptr rtpConfig, + uint16_t maximumFragmentSize = NalUnits::defaultMaximumFragmentSize); + + ChainedOutgoingProduct processOutgoingBinaryMessage(ChainedMessagesProduct messages, + message_ptr control) override; + +private: + const Packetization packetization; + std::shared_ptr sequenceHeader; + + std::vector> packetizeObu(binary_ptr message, uint16_t maximumFragmentSize); +}; + +} // namespace rtc + +#endif /* RTC_ENABLE_MEDIA */ + +#endif /* RTC_AV1_RTP_PACKETIZER_H */ diff --git a/include/rtc/description.hpp b/include/rtc/description.hpp index 123e4b1f7..07e6ea545 100644 --- a/include/rtc/description.hpp +++ b/include/rtc/description.hpp @@ -248,6 +248,7 @@ class RTC_CPP_EXPORT Description { void addH264Codec(int payloadType, optional profile = DEFAULT_H264_VIDEO_PROFILE); void addVP8Codec(int payloadType); void addVP9Codec(int payloadType); + void addAV1Codec(int payloadType); }; bool hasApplication() const; diff --git a/include/rtc/rtc.h b/include/rtc/rtc.h index 7cd44075b..dc5662cdb 100644 --- a/include/rtc/rtc.h +++ b/include/rtc/rtc.h @@ -276,6 +276,12 @@ RTC_C_EXPORT int rtcGetTrackDirection(int tr, rtcDirection *direction); // Media +// Define how OBUs are packetizied in a AV1 Sample +typedef enum { + RTC_OBU_PACKETIZED_OBU = 0, + RTC_OBU_PACKETIZED_TEMPORAL_UNIT = 1, +} rtcObuPacketization; + // Define how NAL units are separated in a H264 sample typedef enum { RTC_NAL_SEPARATOR_LENGTH = 0, // first 4 bytes are NAL unit length diff --git a/include/rtc/rtc.hpp b/include/rtc/rtc.hpp index 75a0ac799..714799846 100644 --- a/include/rtc/rtc.hpp +++ b/include/rtc/rtc.hpp @@ -33,8 +33,9 @@ #include "rtcpreceivingsession.hpp" #include "rtcpsrreporter.hpp" -// Opus/h264 streaming +// Opus/h264/AV1 streaming #include "h264packetizationhandler.hpp" +#include "av1packetizationhandler.hpp" #include "opuspacketizationhandler.hpp" #endif // RTC_ENABLE_MEDIA diff --git a/src/av1packetizationhandler.cpp b/src/av1packetizationhandler.cpp new file mode 100644 index 000000000..0e29d1394 --- /dev/null +++ b/src/av1packetizationhandler.cpp @@ -0,0 +1,20 @@ +/** + * Copyright (c) 2023 Paul-Louis Ageneau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#if RTC_ENABLE_MEDIA + +#include "av1packetizationhandler.hpp" + +namespace rtc { + +AV1PacketizationHandler::AV1PacketizationHandler(shared_ptr packetizer) + : MediaChainableHandler(packetizer) {} + +} // namespace rtc + +#endif /* RTC_ENABLE_MEDIA */ diff --git a/src/av1rtppacketizer.cpp b/src/av1rtppacketizer.cpp new file mode 100644 index 000000000..ebc1e2536 --- /dev/null +++ b/src/av1rtppacketizer.cpp @@ -0,0 +1,230 @@ +/** + * Copyright (c) 2023 Paul-Louis Ageneau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#if RTC_ENABLE_MEDIA + +#include "av1rtppacketizer.hpp" + +#include "impl/internals.hpp" + +namespace rtc { + +const auto payloadHeaderSize = 1; + +const auto zMask = byte(0b10000000); +const auto yMask = byte(0b01000000); +const auto nMask = byte(0b00001000); + +const auto wBitshift = 4; + +const auto obuFrameTypeMask = byte(0b01111000); +const auto obuFrameTypeBitshift = 3; + +const auto obuHeaderSize = 1; +const auto obuHasExtensionMask = byte(0b00000100); +const auto obuHasSizeMask = byte(0b00000010); + +const auto obuFrameTypeSequenceHeader = byte(1); + +const auto obuTemporalUnitDelimiter = std::vector{byte(0x12), byte(0x00)}; + +const auto oneByteLeb128Size = 1; + +const uint8_t sevenLsbBitmask = 0b01111111; +const uint8_t msbBitmask = 0b10000000; + +std::vector extractTemporalUnitObus(binary_ptr message) { + std::vector> obus{}; + + if (message->size() <= 2 || (message->at(0) != obuTemporalUnitDelimiter.at(0)) || + (message->at(1) != obuTemporalUnitDelimiter.at(1))) { + return obus; + } + + size_t messageIndex = 2; + while (messageIndex < message->size()) { + if ((message->at(messageIndex) & obuHasSizeMask) == byte(0)) { + return obus; + } + + if ((message->at(messageIndex) & obuHasExtensionMask) != byte(0)) { + messageIndex++; + } + + // https://aomediacodec.github.io/av1-spec/#leb128 + uint32_t obuLength = 0; + uint8_t leb128Size = 0; + while (leb128Size < 8) { + auto leb128Index = messageIndex + leb128Size + obuHeaderSize; + if (message->size() < leb128Index) { + break; + } + + auto leb128_byte = uint8_t(message->at(leb128Index)); + + obuLength |= ((leb128_byte & sevenLsbBitmask) << (leb128Size * 7)); + leb128Size++; + + if (!(leb128_byte & msbBitmask)) { + break; + } + } + + obus.push_back(std::make_shared(message->begin() + messageIndex, + message->begin() + messageIndex + obuHeaderSize + + leb128Size + obuLength)); + + messageIndex += obuHeaderSize + leb128Size + obuLength; + } + + return obus; +} + +/* + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |Z|Y| W |N|-|-|-| + * +-+-+-+-+-+-+-+-+ + * + * Z: MUST be set to 1 if the first OBU element is an + * OBU fragment that is a continuation of an OBU fragment + * from the previous packet, and MUST be set to 0 otherwise. + * + * Y: MUST be set to 1 if the last OBU element is an OBU fragment + * that will continue in the next packet, and MUST be set to 0 otherwise. + * + * W: two bit field that describes the number of OBU elements in the packet. + * This field MUST be set equal to 0 or equal to the number of OBU elements + * contained in the packet. If set to 0, each OBU element MUST be preceded by + * a length field. If not set to 0 (i.e., W = 1, 2 or 3) the last OBU element + * MUST NOT be preceded by a length field. Instead, the length of the last OBU + * element contained in the packet can be calculated as follows: + * Length of the last OBU element = + * length of the RTP payload + * - length of aggregation header + * - length of previous OBU elements including length fields + * + * N: MUST be set to 1 if the packet is the first packet of a coded video sequence, and MUST be set + * to 0 otherwise. + * + * https://aomediacodec.github.io/av1-rtp-spec/#44-av1-aggregation-header + * + **/ + +std::vector AV1RtpPacketizer::packetizeObu(binary_ptr message, + uint16_t maximumFragmentSize) { + + std::vector> payloads{}; + size_t messageIndex = 0; + + if (message->size() < 1) { + return payloads; + } + + // Cache sequence header and packetize with next OBU + auto frameType = (message->at(0) & obuFrameTypeMask) >> obuFrameTypeBitshift; + if (frameType == obuFrameTypeSequenceHeader) { + sequenceHeader = std::make_shared(message->begin(), message->end()); + return payloads; + } + + size_t messageRemaining = message->size(); + while (messageRemaining > 0) { + auto obuCount = 1; + auto metadataSize = payloadHeaderSize; + + if (sequenceHeader != nullptr) { + obuCount++; + metadataSize += /* 1 byte leb128 */ 1 + int(sequenceHeader->size()); + } + + auto payload = std::make_shared( + std::min(size_t(maximumFragmentSize), messageRemaining + metadataSize)); + auto payloadOffset = payloadHeaderSize; + + payload->at(0) = byte(obuCount) << wBitshift; + + // Packetize cached SequenceHeader + if (obuCount == 2) { + payload->at(0) ^= nMask; + payload->at(1) = byte(sequenceHeader->size() & sevenLsbBitmask); + payloadOffset += oneByteLeb128Size; + + std::memcpy(payload->data() + payloadOffset, sequenceHeader->data(), + sequenceHeader->size()); + payloadOffset += int(sequenceHeader->size()); + + sequenceHeader = nullptr; + } + + // Copy as much of OBU as possible into Payload + auto payloadRemaining = payload->size() - payloadOffset; + std::memcpy(payload->data() + payloadOffset, message->data() + messageIndex, + payloadRemaining); + messageRemaining -= payloadRemaining; + messageIndex += payloadRemaining; + + // Does this Fragment contain an OBU that started in a previous payload + if (payloads.size() > 0) { + payload->at(0) ^= zMask; + } + + // This OBU will be continued in next Payload + if (messageIndex < message->size()) { + payload->at(0) ^= yMask; + } + + payloads.push_back(payload); + } + + return payloads; +} + +AV1RtpPacketizer::AV1RtpPacketizer(AV1RtpPacketizer::Packetization packetization, + shared_ptr rtpConfig, + uint16_t maximumFragmentSize) + : RtpPacketizer(rtpConfig), MediaHandlerRootElement(), maximumFragmentSize(maximumFragmentSize), + packetization(packetization) {} + +ChainedOutgoingProduct +AV1RtpPacketizer::processOutgoingBinaryMessage(ChainedMessagesProduct messages, + message_ptr control) { + ChainedMessagesProduct packets = std::make_shared>(); + for (auto message : *messages) { + std::vector obus; + + if (packetization == AV1RtpPacketizer::Packetization::TemporalUnit) { + obus = extractTemporalUnitObus(message); + } else { + obus.push_back(message); + } + + for (auto obu : obus) { + auto payloads = packetizeObu(obu, maximumFragmentSize); + if (payloads.size() == 0) { + continue; + } + + unsigned i = 0; + for (; i < payloads.size() - 1; i++) { + packets->push_back(packetize(payloads[i], false)); + } + packets->push_back(packetize(payloads[i], true)); + } + } + + if (packets->size() == 0) { + return ChainedOutgoingProduct(); + } + + return {packets, control}; +} + +} // namespace rtc + +#endif /* RTC_ENABLE_MEDIA */ diff --git a/src/description.cpp b/src/description.cpp index 966689137..58b37f4c0 100644 --- a/src/description.cpp +++ b/src/description.cpp @@ -1183,6 +1183,10 @@ void Description::Video::addVP9Codec(int payloadType) { addVideoCodec(payloadType, "VP9", nullopt); } +void Description::Video::addAV1Codec(int pt) { + addVideoCodec(pt, "AV1", nullopt); +} + Description::Type Description::stringToType(const string &typeString) { using TypeMap_t = std::unordered_map; static const TypeMap_t TypeMap = {{"unspec", Type::Unspec},