LibMedia: Add separate classes managing decoding and displaying video

These are unused in this commit, but will later be used to output video
via PlaybackManager, or to decode video directly to some consumer.

Author: Zaggy1024

Libraries/LibMedia/CMakeLists.txt

@@ -11,6 +11,8 @@ set(SOURCES
     Containers/Matroska/MatroskaDemuxer.cpp
     Containers/Matroska/Reader.cpp
     PlaybackManager.cpp
+    Providers/VideoDataProvider.cpp
+    Sinks/DisplayingVideoSink.cpp
     VideoFrame.cpp
 )
 

Libraries/LibMedia/Forward.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -10,10 +11,16 @@ namespace Media {
 
 class CodedFrame;
 class DecoderError;
+class Demuxer;
+class DisplayingVideoSink;
 class FrameQueueItem;
+class MediaTimeProvider;
+class MutexedDemuxer;
 class PlaybackManager;
 class Track;
+class VideoDataProvider;
 class VideoDecoder;
 class VideoFrame;
+class VideoSink;
 
 }

Libraries/LibMedia/MutexedDemuxer.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Forward.h>
+#include <LibMedia/DecoderError.h>
+#include <LibThreading/MutexProtected.h>
+
+#include "Demuxer.h"
+
+namespace Media {
+
+class MutexedDemuxer final : public Demuxer {
+public:
+    MutexedDemuxer(NonnullRefPtr<Demuxer> demuxer)
+        : m_demuxer(move(demuxer))
+    {
+    }
+
+    virtual ~MutexedDemuxer() override
+    {
+        m_demuxer.with_locked([](auto& demuxer) {
+            auto to_destroy = move(demuxer);
+        });
+    }
+
+    virtual DecoderErrorOr<Vector<Track>> get_tracks_for_type(TrackType type) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_tracks_for_type(type);
+        });
+    }
+    virtual DecoderErrorOr<Optional<Track>> get_preferred_track_for_type(TrackType type) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_preferred_track_for_type(type);
+        });
+    }
+
+    virtual DecoderErrorOr<CodedFrame> get_next_sample_for_track(Track track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_next_sample_for_track(track);
+        });
+    }
+
+    virtual DecoderErrorOr<CodecID> get_codec_id_for_track(Track track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_codec_id_for_track(track);
+        });
+    }
+
+    virtual DecoderErrorOr<ReadonlyBytes> get_codec_initialization_data_for_track(Track track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_codec_initialization_data_for_track(track);
+        });
+    }
+
+    virtual DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = {}) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->seek_to_most_recent_keyframe(track, timestamp, earliest_available_sample);
+        });
+    }
+
+    virtual DecoderErrorOr<AK::Duration> duration_of_track(Track const& track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->duration_of_track(track);
+        });
+    }
+    virtual DecoderErrorOr<AK::Duration> total_duration() override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->total_duration();
+        });
+    }
+
+private:
+    Threading::MutexProtected<NonnullRefPtr<Demuxer>> m_demuxer;
+};
+
+}

Libraries/LibMedia/Providers/MediaTimeProvider.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/AtomicRefCounted.h>
+#include <AK/Time.h>
+
+namespace Media {
+
+class MediaTimeProvider : public AtomicRefCounted<MediaTimeProvider> {
+public:
+    virtual ~MediaTimeProvider() = default;
+
+    virtual AK::Duration current_time() const = 0;
+};
+
+}

Libraries/LibMedia/Providers/VideoDataProvider.cpp

@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/EventLoop.h>
+#include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
+#include <LibMedia/MutexedDemuxer.h>
+#include <LibMedia/Providers/MediaTimeProvider.h>
+#include <LibMedia/Sinks/VideoSink.h>
+#include <LibMedia/VideoDecoder.h>
+#include <LibMedia/VideoFrame.h>
+#include <LibThreading/Thread.h>
+
+#include "VideoDataProvider.h"
+
+namespace Media {
+
+DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> VideoDataProvider::try_create(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, RefPtr<MediaTimeProvider> const& time_provider)
+{
+    auto codec_id = TRY(demuxer->get_codec_id_for_track(track));
+    auto codec_initialization_data = TRY(demuxer->get_codec_initialization_data_for_track(track));
+    auto decoder = DECODER_TRY_ALLOC(FFmpeg::FFmpegVideoDecoder::try_create(codec_id, codec_initialization_data));
+
+    auto thread_data = DECODER_TRY_ALLOC(try_make_ref_counted<VideoDataProvider::ThreadData>(demuxer, track, move(decoder), time_provider));
+    auto provider = DECODER_TRY_ALLOC(try_make_ref_counted<VideoDataProvider>(thread_data));
+
+    auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int {
+        while (!thread_data->should_thread_exit())
+            thread_data->push_data_and_decode_some_frames();
+        return 0;
+    }));
+    thread->start();
+    thread->detach();
+
+    return provider;
+}
+
+DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> VideoDataProvider::try_create(NonnullRefPtr<Demuxer> const& demuxer, Track const& track, RefPtr<MediaTimeProvider> const& time_provider)
+{
+    auto mutexed_demuxer = DECODER_TRY_ALLOC(try_make_ref_counted<MutexedDemuxer>(demuxer));
+    return try_create(mutexed_demuxer, track, time_provider);
+}
+
+VideoDataProvider::VideoDataProvider(NonnullRefPtr<ThreadData> const& thread_state)
+    : m_thread_data(thread_state)
+{
+}
+
+VideoDataProvider::~VideoDataProvider()
+{
+    m_thread_data->exit();
+}
+
+void VideoDataProvider::set_error_handler(ErrorHandler&& handler)
+{
+    m_thread_data->set_error_handler(move(handler));
+}
+
+TimedImage VideoDataProvider::retrieve_frame()
+{
+    auto locker = m_thread_data->take_lock();
+    if (m_thread_data->queue().is_empty())
+        return TimedImage();
+    auto result = m_thread_data->queue().dequeue();
+    m_thread_data->wake();
+    return result;
+}
+
+void VideoDataProvider::seek(AK::Duration timestamp)
+{
+    m_thread_data->seek(timestamp);
+}
+
+VideoDataProvider::ThreadData::ThreadData(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, NonnullOwnPtr<VideoDecoder>&& decoder, RefPtr<MediaTimeProvider> const& time_provider)
+    : m_main_thread_event_loop(Core::EventLoop::current())
+    , m_demuxer(demuxer)
+    , m_track(track)
+    , m_decoder(move(decoder))
+    , m_time_provider(time_provider)
+{
+}
+
+VideoDataProvider::ThreadData::~ThreadData() = default;
+
+void VideoDataProvider::ThreadData::set_error_handler(ErrorHandler&& handler)
+{
+    auto locker = take_lock();
+    m_error_handler = move(handler);
+    m_wait_condition.broadcast();
+}
+
+void VideoDataProvider::ThreadData::exit()
+{
+    m_exit = true;
+    m_wait_condition.broadcast();
+}
+
+VideoDataProvider::ImageQueue& VideoDataProvider::ThreadData::queue()
+{
+    return m_queue;
+}
+
+void VideoDataProvider::ThreadData::seek(AK::Duration timestamp)
+{
+    auto seek_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
+    if (seek_result.is_error()) {
+        m_error_handler(seek_result.release_error());
+    } else {
+        auto locker = take_lock();
+        m_is_in_error_state = false;
+        m_wait_condition.broadcast();
+    }
+}
+
+bool VideoDataProvider::ThreadData::should_thread_exit() const
+{
+    return m_exit;
+}
+
+void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
+{
+    // FIXME: Check if the PlaybackManager's current time is ahead of the next keyframe, and seek to it if so.
+    //        Demuxers currently can't report the next keyframe in a convenient way, so that will need implementing
+    //        before this functionality can exist.
+
+    auto set_error_and_wait_for_seek = [this](DecoderError&& error) {
+        auto locker = take_lock();
+        m_is_in_error_state = true;
+        while (!m_error_handler)
+            m_wait_condition.wait();
+        m_main_thread_event_loop.deferred_invoke([this, error = move(error)] mutable {
+            m_error_handler(move(error));
+        });
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Video Data Provider: Encountered an error, waiting for a seek to start decoding again...");
+        while (m_is_in_error_state)
+            m_wait_condition.wait();
+    };
+
+    auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
+    if (sample_result.is_error()) {
+        // FIXME: Handle the end of the stream.
+        set_error_and_wait_for_seek(sample_result.release_error());
+        return;
+    }
+
+    auto coded_frame = sample_result.release_value();
+    auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data());
+    if (decode_result.is_error()) {
+        set_error_and_wait_for_seek(decode_result.release_error());
+        return;
+    }
+
+    while (true) {
+        auto frame_result = m_decoder->get_decoded_frame();
+        if (frame_result.is_error()) {
+            if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
+                break;
+            set_error_and_wait_for_seek(frame_result.release_error());
+            break;
+        }
+
+        auto frame = frame_result.release_value();
+
+        // Convert the frame for display.
+        auto& cicp = frame->cicp();
+        auto container_cicp = coded_frame.auxiliary_data().get<CodedVideoFrameData>().container_cicp();
+        cicp.adopt_specified_values(container_cicp);
+        cicp.default_code_points_if_unspecified({ ColorPrimaries::BT709, TransferCharacteristics::BT709, MatrixCoefficients::BT709, VideoFullRangeFlag::Studio });
+
+        // BT.470 M, B/G, BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
+        // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
+        // handling those as sRGB instead, which causes no transfer function change in the output,
+        // unless display color management is later implemented.
+        switch (cicp.transfer_characteristics()) {
+        case TransferCharacteristics::BT470BG:
+        case TransferCharacteristics::BT470M:
+        case TransferCharacteristics::BT601:
+        case TransferCharacteristics::BT709:
+        case TransferCharacteristics::BT2020BitDepth10:
+        case TransferCharacteristics::BT2020BitDepth12:
+            cicp.set_transfer_characteristics(TransferCharacteristics::SRGB);
+            break;
+        default:
+            break;
+        }
+
+        auto bitmap_result = frame->to_bitmap();
+
+        if (bitmap_result.is_error()) {
+            set_error_and_wait_for_seek(bitmap_result.release_error());
+            return;
+        }
+
+        {
+            auto locker = take_lock();
+            while (m_queue.size() >= m_queue_max_size) {
+                m_wait_condition.wait();
+                if (should_thread_exit())
+                    return;
+            }
+            m_queue.enqueue(TimedImage(frame->timestamp(), bitmap_result.release_value()));
+        }
+    }
+}
+
+}