Skip to content

Commit 0ff330c

Browse files
Zaggy1024gmta
Zaggy1024
authored and
gmta
committed
LibMedia: Play audio through PlaybackManager using Providers/Sinks
This commit implements the functionality to play back audio through PlaybackManager. To decode the audio data, AudioDataProviders are created for each track in the provided media data. These providers will fill their audio block queue, then sit idle until their corresponding tracks are enabled. In order to output the audio, one AudioMixingSink is created which manages a PlaybackStream which requests audio blocks from multiple AudioDataProviders and mixes them into one buffer with sample-perfect precision.
1 parent dd05283 commit 0ff330c

File tree

14 files changed

+1031
-5
lines changed

14 files changed

+1031
-5
lines changed

‎Libraries/LibMedia/AudioBlock.h‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
3+
*
4+
* SPDX-License-Identifier: BSD-2-Clause
5+
*/
6+
7+
#pragma once
8+
9+
#include <AK/FixedArray.h>
10+
#include <AK/Time.h>
11+
12+
namespace Media {
13+
14+
class AudioBlock {
15+
public:
16+
using Data = FixedArray<float>;
17+
18+
u32 sample_rate() const { return m_sample_rate; }
19+
u8 channel_count() const { return m_channel_count; }
20+
AK::Duration start_timestamp() const { return m_start_timestamp; }
21+
Data& data() { return m_data; }
22+
Data const& data() const { return m_data; }
23+
24+
void clear()
25+
{
26+
m_sample_rate = 0;
27+
m_channel_count = 0;
28+
m_start_timestamp = AK::Duration::zero();
29+
m_data = Data();
30+
}
31+
template<typename Callback>
32+
void emplace(u32 sample_rate, u8 channel_count, AK::Duration start_timestamp, Callback data_callback)
33+
{
34+
VERIFY(sample_rate != 0);
35+
VERIFY(channel_count != 0);
36+
VERIFY(m_data.is_empty());
37+
m_sample_rate = sample_rate;
38+
m_channel_count = channel_count;
39+
m_start_timestamp = start_timestamp;
40+
data_callback(m_data);
41+
}
42+
bool is_empty() const
43+
{
44+
return m_sample_rate == 0;
45+
}
46+
size_t data_count() const
47+
{
48+
return data().size();
49+
}
50+
size_t sample_count() const
51+
{
52+
return data_count() / m_channel_count;
53+
}
54+
55+
private:
56+
u32 m_sample_rate { 0 };
57+
u8 m_channel_count { 0 };
58+
AK::Duration m_start_timestamp;
59+
Data m_data;
60+
};
61+
62+
}

‎Libraries/LibMedia/AudioDecoder.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
3+
*
4+
* SPDX-License-Identifier: BSD-2-Clause
5+
*/
6+
7+
#pragma once
8+
9+
#include <AK/ByteBuffer.h>
10+
#include <AK/NonnullOwnPtr.h>
11+
#include <AK/Time.h>
12+
#include <AK/Vector.h>
13+
#include <LibMedia/AudioBlock.h>
14+
#include <LibMedia/DecoderError.h>
15+
16+
namespace Media {
17+
18+
class AudioDecoder {
19+
public:
20+
virtual ~AudioDecoder() { }
21+
22+
virtual DecoderErrorOr<void> receive_coded_data(AK::Duration timestamp, ReadonlyBytes coded_data) = 0;
23+
// Writes all buffered audio samples to the provided block.
24+
virtual DecoderErrorOr<void> write_next_block(AudioBlock&) = 0;
25+
26+
virtual void flush() = 0;
27+
};
28+
29+
}

‎Libraries/LibMedia/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ set(SOURCES
1111
Containers/Matroska/MatroskaDemuxer.cpp
1212
Containers/Matroska/Reader.cpp
1313
PlaybackManager.cpp
14+
Providers/AudioDataProvider.cpp
1415
Providers/VideoDataProvider.cpp
16+
Sinks/AudioMixingSink.cpp
1517
Sinks/DisplayingVideoSink.cpp
1618
VideoFrame.cpp
1719
)
@@ -21,6 +23,7 @@ target_link_libraries(LibMedia PRIVATE LibCore LibCrypto LibIPC LibGfx LibThread
2123

2224
target_sources(LibMedia PRIVATE
2325
Audio/FFmpegLoader.cpp
26+
FFmpeg/FFmpegAudioDecoder.cpp
2427
FFmpeg/FFmpegDemuxer.cpp
2528
FFmpeg/FFmpegIOContext.cpp
2629
FFmpeg/FFmpegVideoDecoder.cpp

‎Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.cpp‎

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
/*
2+
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
3+
*
4+
* SPDX-License-Identifier: BSD-2-Clause
5+
*/
6+
7+
#include <LibCore/System.h>
8+
#include <LibMedia/AudioBlock.h>
9+
#include <LibMedia/FFmpeg/FFmpegHelpers.h>
10+
11+
#include "FFmpegAudioDecoder.h"
12+
13+
namespace Media::FFmpeg {
14+
15+
DecoderErrorOr<NonnullOwnPtr<FFmpegAudioDecoder>> FFmpegAudioDecoder::try_create(CodecID codec_id, ReadonlyBytes codec_initialization_data)
16+
{
17+
AVCodecContext* codec_context = nullptr;
18+
AVPacket* packet = nullptr;
19+
AVFrame* frame = nullptr;
20+
ArmedScopeGuard memory_guard {
21+
[&] {
22+
avcodec_free_context(&codec_context);
23+
av_packet_free(&packet);
24+
av_frame_free(&frame);
25+
}
26+
};
27+
28+
auto ff_codec_id = ffmpeg_codec_id_from_media_codec_id(codec_id);
29+
auto const* codec = avcodec_find_decoder(ff_codec_id);
30+
if (!codec)
31+
return DecoderError::format(DecoderErrorCategory::NotImplemented, "Could not find FFmpeg decoder for codec {}", codec_id);
32+
33+
codec_context = avcodec_alloc_context3(codec);
34+
if (!codec_context)
35+
return DecoderError::format(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg codec context for codec {}", codec_id);
36+
37+
codec_context->time_base = { 1, 1'000'000 };
38+
codec_context->thread_count = static_cast<int>(min(Core::System::hardware_concurrency(), 4));
39+
40+
if (!codec_initialization_data.is_empty()) {
41+
if (codec_initialization_data.size() > NumericLimits<int>::max())
42+
return DecoderError::corrupted("Codec initialization data is too large"sv);
43+
44+
codec_context->extradata = static_cast<u8*>(av_malloc(codec_initialization_data.size() + AV_INPUT_BUFFER_PADDING_SIZE));
45+
if (!codec_context->extradata)
46+
return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate codec initialization data buffer for FFmpeg codec"sv);
47+
48+
memcpy(codec_context->extradata, codec_initialization_data.data(), codec_initialization_data.size());
49+
codec_context->extradata_size = static_cast<int>(codec_initialization_data.size());
50+
}
51+
52+
if (avcodec_open2(codec_context, codec, nullptr) < 0)
53+
return DecoderError::format(DecoderErrorCategory::Unknown, "Unknown error occurred when opening FFmpeg codec {}", codec_id);
54+
55+
packet = av_packet_alloc();
56+
if (!packet)
57+
return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg packet"sv);
58+
59+
frame = av_frame_alloc();
60+
if (!frame)
61+
return DecoderError::with_description(DecoderErrorCategory::Memory, "Failed to allocate FFmpeg frame"sv);
62+
63+
memory_guard.disarm();
64+
return DECODER_TRY_ALLOC(try_make<FFmpegAudioDecoder>(codec_context, packet, frame));
65+
}
66+
67+
FFmpegAudioDecoder::FFmpegAudioDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame)
68+
: m_codec_context(codec_context)
69+
, m_packet(packet)
70+
, m_frame(frame)
71+
{
72+
}
73+
74+
FFmpegAudioDecoder::~FFmpegAudioDecoder()
75+
{
76+
av_packet_free(&m_packet);
77+
av_frame_free(&m_frame);
78+
avcodec_free_context(&m_codec_context);
79+
}
80+
81+
DecoderErrorOr<void> FFmpegAudioDecoder::receive_coded_data(AK::Duration timestamp, ReadonlyBytes coded_data)
82+
{
83+
VERIFY(coded_data.size() < NumericLimits<int>::max());
84+
85+
m_packet->data = const_cast<u8*>(coded_data.data());
86+
m_packet->size = static_cast<int>(coded_data.size());
87+
m_packet->pts = timestamp.to_microseconds();
88+
m_packet->dts = m_packet->pts;
89+
90+
auto result = avcodec_send_packet(m_codec_context, m_packet);
91+
switch (result) {
92+
case 0:
93+
return {};
94+
case AVERROR(EAGAIN):
95+
return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder cannot decode any more data until frames have been retrieved"sv);
96+
case AVERROR_EOF:
97+
return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
98+
case AVERROR(EINVAL):
99+
return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
100+
case AVERROR(ENOMEM):
101+
return DecoderError::with_description(DecoderErrorCategory::Memory, "FFmpeg codec ran out of internal memory"sv);
102+
default:
103+
return DecoderError::with_description(DecoderErrorCategory::Corrupted, "FFmpeg codec reports that the data is corrupted"sv);
104+
}
105+
}
106+
107+
template<typename T>
108+
static float float_sample_from_frame_data(u8** data, size_t plane, size_t index);
109+
110+
template<>
111+
float float_sample_from_frame_data<u8>(u8** data, size_t plane, size_t index)
112+
{
113+
return static_cast<float>(data[plane][index] - 127) / 255;
114+
}
115+
116+
template<typename T>
117+
requires(IsSigned<T>)
118+
static float float_sample_from_frame_data(u8** data, size_t plane, size_t index)
119+
{
120+
auto* pointer = reinterpret_cast<T*>(data[plane]);
121+
return pointer[index] / static_cast<float>(NumericLimits<T>::max());
122+
}
123+
124+
template<typename T>
125+
requires(IsFloatingPoint<T>)
126+
static float float_sample_from_frame_data(u8** data, size_t plane, size_t index)
127+
{
128+
auto* pointer = reinterpret_cast<T*>(data[plane]);
129+
return pointer[index];
130+
}
131+
132+
DecoderErrorOr<void> FFmpegAudioDecoder::write_next_block(AudioBlock& block)
133+
{
134+
auto result = avcodec_receive_frame(m_codec_context, m_frame);
135+
136+
switch (result) {
137+
case 0: {
138+
auto timestamp = AK::Duration::from_microseconds(m_frame->pts);
139+
140+
if (m_frame->ch_layout.nb_channels > 2)
141+
return DecoderError::not_implemented();
142+
143+
VERIFY(m_frame->sample_rate > 0);
144+
VERIFY(m_frame->ch_layout.nb_channels > 0);
145+
146+
block.emplace(m_frame->sample_rate, m_frame->ch_layout.nb_channels, timestamp, [&](AudioBlock::Data& data) {
147+
auto format = static_cast<AVSampleFormat>(m_frame->format);
148+
auto is_planar = av_sample_fmt_is_planar(format) != 0;
149+
auto planar_format = av_get_planar_sample_fmt(format);
150+
151+
VERIFY(m_frame->nb_samples >= 0);
152+
auto sample_count = static_cast<size_t>(m_frame->nb_samples);
153+
auto channel_count = static_cast<size_t>(m_frame->ch_layout.nb_channels);
154+
auto count = sample_count * channel_count;
155+
data = MUST(AudioBlock::Data::create(count));
156+
157+
auto sample_size = [&] {
158+
switch (planar_format) {
159+
case AV_SAMPLE_FMT_U8P:
160+
return sizeof(u8);
161+
case AV_SAMPLE_FMT_S16P:
162+
return sizeof(i16);
163+
case AV_SAMPLE_FMT_S32P:
164+
return sizeof(i32);
165+
case AV_SAMPLE_FMT_FLTP:
166+
return sizeof(float);
167+
case AV_SAMPLE_FMT_DBLP:
168+
return sizeof(double);
169+
case AV_SAMPLE_FMT_S64P:
170+
return sizeof(i64);
171+
default:
172+
VERIFY_NOT_REACHED();
173+
}
174+
}();
175+
176+
VERIFY(m_frame->linesize[0] > 0);
177+
if (is_planar)
178+
VERIFY(static_cast<size_t>(m_frame->linesize[0]) >= sample_count * sample_size);
179+
else
180+
VERIFY(static_cast<size_t>(m_frame->linesize[0]) >= sample_count * channel_count * sample_size);
181+
182+
for (size_t i = 0; i < count; i++) {
183+
size_t plane = 0;
184+
size_t index_in_plane = i;
185+
if (is_planar) {
186+
plane = i % channel_count;
187+
index_in_plane = i / channel_count;
188+
}
189+
190+
auto float_sample = [&] {
191+
switch (planar_format) {
192+
case AV_SAMPLE_FMT_U8P:
193+
return float_sample_from_frame_data<u8>(m_frame->extended_data, plane, index_in_plane);
194+
case AV_SAMPLE_FMT_S16P:
195+
return float_sample_from_frame_data<i16>(m_frame->extended_data, plane, index_in_plane);
196+
case AV_SAMPLE_FMT_S32P:
197+
return float_sample_from_frame_data<i32>(m_frame->extended_data, plane, index_in_plane);
198+
case AV_SAMPLE_FMT_FLTP:
199+
return float_sample_from_frame_data<float>(m_frame->extended_data, plane, index_in_plane);
200+
case AV_SAMPLE_FMT_DBLP:
201+
return float_sample_from_frame_data<double>(m_frame->extended_data, plane, index_in_plane);
202+
case AV_SAMPLE_FMT_S64P:
203+
return float_sample_from_frame_data<i64>(m_frame->extended_data, plane, index_in_plane);
204+
default:
205+
VERIFY_NOT_REACHED();
206+
}
207+
}();
208+
data[i] = float_sample;
209+
}
210+
});
211+
212+
return {};
213+
}
214+
case AVERROR(EAGAIN):
215+
return DecoderError::with_description(DecoderErrorCategory::NeedsMoreInput, "FFmpeg decoder has no frames available, send more input"sv);
216+
case AVERROR_EOF:
217+
return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "FFmpeg decoder has been flushed"sv);
218+
case AVERROR(EINVAL):
219+
return DecoderError::with_description(DecoderErrorCategory::Invalid, "FFmpeg codec has not been opened"sv);
220+
default:
221+
return DecoderError::format(DecoderErrorCategory::Unknown, "FFmpeg codec encountered an unexpected error retrieving frames with code {:x}", result);
222+
}
223+
}
224+
225+
void FFmpegAudioDecoder::flush()
226+
{
227+
avcodec_flush_buffers(m_codec_context);
228+
}
229+
230+
}

‎Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
3+
*
4+
* SPDX-License-Identifier: BSD-2-Clause
5+
*/
6+
7+
#pragma once
8+
9+
#include <LibMedia/AudioDecoder.h>
10+
#include <LibMedia/CodecID.h>
11+
#include <LibMedia/Export.h>
12+
#include <LibMedia/FFmpeg/FFmpegForward.h>
13+
14+
namespace Media::FFmpeg {
15+
16+
class MEDIA_API FFmpegAudioDecoder final : public AudioDecoder {
17+
public:
18+
static DecoderErrorOr<NonnullOwnPtr<FFmpegAudioDecoder>> try_create(CodecID, ReadonlyBytes codec_initialization_data);
19+
FFmpegAudioDecoder(AVCodecContext* codec_context, AVPacket* packet, AVFrame* frame);
20+
virtual ~FFmpegAudioDecoder() override;
21+
22+
virtual DecoderErrorOr<void> receive_coded_data(AK::Duration timestamp, ReadonlyBytes coded_data) override;
23+
// Writes all buffered audio samples to the provided block.
24+
virtual DecoderErrorOr<void> write_next_block(AudioBlock&) override;
25+
26+
virtual void flush() override;
27+
28+
private:
29+
AVCodecContext* m_codec_context;
30+
AVPacket* m_packet;
31+
AVFrame* m_frame;
32+
};
33+
34+
}

‎Libraries/LibMedia/Forward.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
namespace Media {
1111

1212
class CodedFrame;
13+
class AudioDataProvider;
14+
class AudioDecoder;
15+
class AudioMixingSink;
16+
class AudioSink;
1317
class DecoderError;
1418
class Demuxer;
1519
class DisplayingVideoSink;

0 commit comments

Comments
 (0)