Skip to content

Commit

Permalink
[MSE][GStreamer] Support Google Dynamic Ad Insertion (DAI)
Browse files Browse the repository at this point in the history 
https://bugs.webkit.org/show_bug.cgi?id=216039
<rdar://problem/68638316>

Patch by Philippe Normand <philn@igalia.com> on 2022-05-25
Reviewed by Alicia Boya Garcia.

The "simple DAI" demo was not working as expected because the new source pad of the demuxer in the append
pipeline was sinking buffers to a "black hole". That was happening because the second time the
no-more-pads signal was emitted, the previous source pad was still linked to a track, and there was
no code to unlink/relink demuxer pads. This is now handled...

The specific "simple DAI" demo generates 2 AAC streams, with different tkhd track-id fields, we had
no layout test reproducing this use-case, so this patch provides one.

Test: media-source/media-source-audio-track-id-switch.html

* LayoutTests/media/media-source/content/test-48kHz-new-track-id.m4a: Added.
* LayoutTests/media/media-source/content/test-48khz-new-track-id-manifest.json: Added.
* LayoutTests/media/media-source/media-source-audio-track-id-switch-expected.txt: Added.
* LayoutTests/media/media-source/media-source-audio-track-id-switch.html: Added.
* Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.cpp:
(WebCore::AppendPipeline::handleErrorSyncMessage):
(WebCore::AppendPipeline::didReceiveInitializationSegment):
(WebCore::createOptionalParserForFormat):
(WebCore::AppendPipeline::recycleTrackForPad):
(WebCore::AppendPipeline::linkPadWithTrack):
(WebCore::AppendPipeline::tryMatchPadToExistingTrack): Deleted.
* Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.h:
(WebCore::AppendPipeline::Track::isLinked const):

Canonical link: https://commits.webkit.org/250950@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@294791 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
@philn @webkit-commit-queue
philn authored and webkit-commit-queue committed May 25, 2022
1 parent 7003d0d commit 296d132
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 22 deletions.
Binary file added LayoutTests/media/media-source/content/test-48kHz-new-track-id.m4a
View file
Binary file not shown.
19 changes: 19 additions & 0 deletions LayoutTests/media/media-source/content/test-48khz-new-track-id-manifest.json
View file
@@ -0,0 +1,19 @@
{
"url": "content/test-48kHz-new-track-id.m4a",
"type": "audio/mp4; codecs=\"mp4a.40.2\"",
"init": { "offset": 0, "size": 624 },
"duration": 10.0906,
"media": [
{ "offset": 624, "size": 1308, "timestamp": 0, "duration": 1.0027},
{ "offset": 1932, "size": 1491, "timestamp": 1.0027, "duration": 1.0027},
{ "offset": 3423, "size": 1341, "timestamp": 2.0053, "duration": 1.0027},
{ "offset": 4764, "size": 1563, "timestamp": 3.008, "duration": 0.9813},
{ "offset": 6327, "size": 1425, "timestamp": 3.9893, "duration": 1.0027},
{ "offset": 7752, "size": 1770, "timestamp": 4.992, "duration": 1.0027},
{ "offset": 9522, "size": 1910, "timestamp": 5.9947, "duration": 1.0027},
{ "offset": 11432, "size": 1887, "timestamp": 6.9973, "duration": 1.0027},
{ "offset": 13319, "size": 1919, "timestamp": 8, "duration": 1.0027},
{ "offset": 15238, "size": 1892, "timestamp": 9.0027, "duration": 1.0027},
{ "offset": 17130, "size": 1231, "timestamp": 10.0053, "duration": 0.0853}
]
}
12 changes: 12 additions & 0 deletions LayoutTests/media/media-source/media-source-audio-track-id-switch-expected.txt
View file
@@ -0,0 +1,12 @@

RUN(audio.src = URL.createObjectURL(source))
EVENT(sourceopen)
RUN(source.duration = loader.duration())
RUN(sourceBuffer = source.addSourceBuffer(loader.type()))
RUN(sourceBuffer.appendBuffer(loader.initSegment()))
EVENT(updateend)
Switching to a similar audio stream but with different mp4 track ID.
RUN(sourceBuffer.appendBuffer(loader.initSegment()))
EVENT(updateend)
END OF TEST

55 changes: 55 additions & 0 deletions LayoutTests/media/media-source/media-source-audio-track-id-switch.html
View file
@@ -0,0 +1,55 @@
<!DOCTYPE html>
<html>
<head>
<title>media-source-audio-track-id-switch</title>
<script src="media-source-loader.js"></script>
<script src="../video-test.js"></script>
<script>
var loader;
var source;
var sourceBuffer;
var audio;

function runTest() {
audio = document.getElementsByTagName('audio')[0];

loader = new MediaSourceLoader('content/test-48khz-manifest.json');
loader.onload = mediaDataLoaded;
loader.onerror = mediaDataLoadingFailed;
}

function mediaDataLoadingFailed() {
failTest('Media data loading failed');
}

function mediaDataLoaded() {
source = new MediaSource();
waitForEvent('sourceopen', sourceOpen, false, false, source);
run('audio.src = URL.createObjectURL(source)');
}

function sourceOpen() {
run('source.duration = loader.duration()');
run('sourceBuffer = source.addSourceBuffer(loader.type())');
sourceBuffer.addEventListener('error', mediaDataLoadingFailed);
waitForEventOn(sourceBuffer, 'updateend', sourceInitialized, false, true);
run('sourceBuffer.appendBuffer(loader.initSegment())');
}

function sourceInitialized() {
consoleWrite('Switching to a similar audio stream but with different mp4 track ID.')
loader = new MediaSourceLoader('content/test-48khz-new-track-id-manifest.json');
loader.onload = newTrackMediaDataLoaded;
loader.onerror = mediaDataLoadingFailed;
}

function newTrackMediaDataLoaded() {
waitForEventOn(sourceBuffer, 'updateend', endTest, false, true);
run('sourceBuffer.appendBuffer(loader.initSegment())');
}
</script>
</head>
<body onload="runTest()">
<audio controls/>
</body>
</html>
97 changes: 76 additions & 21 deletions Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.cpp
View file
Expand Up @@ -261,6 +261,7 @@ void AppendPipeline::handleErrorSyncMessage(GstMessage* message)
ASSERT(!isMainThread());
GST_WARNING_OBJECT(m_pipeline.get(), "Demuxing error: %" GST_PTR_FORMAT, message);
handleErrorConditionFromStreamingThread();
GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(m_pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "demuxing-error");
}

GstPadProbeReturn AppendPipeline::appsrcEndOfAppendCheckerProbe(GstPadProbeInfo* padProbeInfo)
Expand Down Expand Up @@ -453,22 +454,49 @@ void AppendPipeline::didReceiveInitializationSegment()
trackIndex++;
}
} else {
// Link pads to existing Track objects that don't have a linked pad yet.
unsigned countPads = 0;
// Since we don't rely on the demuxer pad-added signal and this pipeline is not
// stream-aware, we need to account for stream topology changes ourselves.
unsigned videoPadsCount = 0;
unsigned audioPadsCount = 0;
unsigned textPadsCount = 0;
for (auto pad : GstIteratorAdaptor<GstPad>(GUniquePtr<GstIterator>(gst_element_iterate_src_pads(m_demux.get())))) {
if (gst_pad_is_linked(pad))
continue;
auto [parsedCaps, streamType, presentationSize] = parseDemuxerSrcPadCaps(adoptGRef(gst_pad_get_current_caps(pad)).get());
if (streamType == StreamType::Audio)
audioPadsCount++;
else if (streamType == StreamType::Video)
videoPadsCount++;
else if (streamType == StreamType::Text)
textPadsCount++;
}

unsigned videoTracksCount = 0;
unsigned audioTracksCount = 0;
unsigned textTracksCount = 0;
for (const auto& track : m_tracks) {
if (track->streamType == StreamType::Audio)
audioTracksCount++;
else if (track->streamType == StreamType::Video)
videoTracksCount++;
else if (track->streamType == StreamType::Text)
textTracksCount++;
}

if (videoPadsCount < videoTracksCount || audioPadsCount < audioTracksCount || textPadsCount < textTracksCount) {
GST_WARNING_OBJECT(pipeline(), "New demuxed stream topology doesn't match the existing tracks topology");
m_sourceBufferPrivate.appendParsingFailed();
return;
}

// Link pads to existing Track objects that don't have a linked pad yet. Existing linked
// tracks are recycled if their stream type matches the new demuxer source pads.
for (GstPad* pad : GstIteratorAdaptor<GstPad>(GUniquePtr<GstIterator>(gst_element_iterate_src_pads(m_demux.get())))) {
countPads++;
Track* track = tryMatchPadToExistingTrack(pad);
if (!track) {
if (!recycleTrackForPad(pad)) {
GST_WARNING_OBJECT(pipeline(), "Can't match pad to existing tracks in the AppendPipeline: %" GST_PTR_FORMAT, pad);
m_sourceBufferPrivate.appendParsingFailed();
return;
}
linkPadWithTrack(pad, *track);
}
if (countPads != m_tracks.size()) {
GST_WARNING_OBJECT(pipeline(), "Number of pads (%u) doesn't match number of tracks (%zu).", countPads, m_tracks.size());
m_sourceBufferPrivate.appendParsingFailed();
return;
}
}

Expand Down Expand Up @@ -642,8 +670,8 @@ createOptionalParserForFormat(const AtomString& trackId, const GstCaps* caps)
{
GstStructure* structure = gst_caps_get_structure(caps, 0);
const char* mediaType = gst_structure_get_name(structure);
GUniquePtr<char> parserName(g_strdup_printf("%s_parser", trackId.string().utf8().data()));
const gchar* elementClass = "identity";
auto parserName = makeString(trackId, "_parser"_s);
const char* elementClass = "identity";

if (!g_strcmp0(mediaType, "audio/x-opus"))
elementClass = "opusparse";
Expand All @@ -660,13 +688,12 @@ createOptionalParserForFormat(const AtomString& trackId, const GstCaps* caps)
case 4:
elementClass = "aacparse";
break;
default: {
GUniquePtr<char> capsString(gst_caps_to_string(caps));
GST_WARNING("Unsupported audio mpeg caps: %s", capsString.get());
}
default:
GST_WARNING("Unsupported audio mpeg caps: %" GST_PTR_FORMAT, caps);
}
}
return GRefPtr<GstElement>(makeGStreamerElement(elementClass, parserName.get()));
GST_DEBUG("Creating %s parser for stream with caps %" GST_PTR_FORMAT, elementClass, caps);
return GRefPtr<GstElement>(makeGStreamerElement(elementClass, parserName.ascii().data()));
}

AtomString AppendPipeline::generateTrackId(StreamType streamType, int padIndex)
Expand Down Expand Up @@ -725,18 +752,20 @@ std::pair<AppendPipeline::CreateTrackResult, AppendPipeline::Track*> AppendPipel
return { CreateTrackResult::TrackCreated, &track };
}

AppendPipeline::Track* AppendPipeline::tryMatchPadToExistingTrack(GstPad *demuxerSrcPad)
bool AppendPipeline::recycleTrackForPad(GstPad* demuxerSrcPad)
{
ASSERT(isMainThread());
ASSERT(m_hasReceivedFirstInitializationSegment);
auto trackId = AtomString::fromLatin1(GST_PAD_NAME(demuxerSrcPad));
auto [parsedCaps, streamType, presentationSize] = parseDemuxerSrcPadCaps(adoptGRef(gst_pad_get_current_caps(demuxerSrcPad)).get());

GST_DEBUG_OBJECT(demuxerSrcPad, "Caps: %" GST_PTR_FORMAT, parsedCaps.get());

// Try to find a matching pre-existing track. Ideally, tracks should be matched by track ID, but matching by type
// is provided as a fallback -- which will be used, since we don't have a way to fetch those from GStreamer at the moment.
Track* matchingTrack = nullptr;
for (std::unique_ptr<Track>& track : m_tracks) {
if (track->streamType != streamType || gst_pad_is_linked(track->entryPad.get()))
if (track->streamType != streamType)
continue;
matchingTrack = &*track;
if (track->trackId == trackId)
Expand All @@ -748,12 +777,38 @@ AppendPipeline::Track* AppendPipeline::tryMatchPadToExistingTrack(GstPad *demuxe
GST_WARNING_OBJECT(pipeline(), "Couldn't find a matching pre-existing track for pad '%s' with parsed caps %" GST_PTR_FORMAT
" on non-first initialization segment, will be connected to a black hole probe.", GST_PAD_NAME(demuxerSrcPad), parsedCaps.get());
gst_pad_add_probe(demuxerSrcPad, GST_PAD_PROBE_TYPE_BUFFER, reinterpret_cast<GstPadProbeCallback>(appendPipelineDemuxerBlackHolePadProbe), nullptr, nullptr);
return false;
}

if (!matchingTrack->isLinked())
linkPadWithTrack(demuxerSrcPad, *matchingTrack);
else {
// Unlink from old track and link to new track, by 1. stopping parser/sink, 2. unlinking
// demuxer from track, 3. restarting parser/sink.
if (matchingTrack->parser)
gst_element_set_state(matchingTrack->parser.get(), GST_STATE_NULL);
gst_element_set_state(matchingTrack->appsink.get(), GST_STATE_NULL);

auto peer = adoptGRef(gst_pad_get_peer(matchingTrack->entryPad.get()));
if (peer.get() != demuxerSrcPad) {
GST_DEBUG_OBJECT(peer.get(), "Unlinking from track %s", matchingTrack->trackId.string().ascii().data());
gst_pad_unlink(peer.get(), matchingTrack->entryPad.get());
linkPadWithTrack(demuxerSrcPad, *matchingTrack);
matchingTrack->caps = WTFMove(parsedCaps);
matchingTrack->presentationSize = presentationSize;
} else
GST_DEBUG_OBJECT(pipeline(), "%s track pads match, nothing to re-link", matchingTrack->trackId.string().ascii().data());

gst_element_set_state(matchingTrack->appsink.get(), GST_STATE_PLAYING);
if (matchingTrack->parser)
gst_element_set_state(matchingTrack->parser.get(), GST_STATE_PLAYING);
}
return matchingTrack;
return true;
}

void AppendPipeline::linkPadWithTrack(GstPad* demuxerSrcPad, Track& track)
{
GST_DEBUG_OBJECT(demuxerSrcPad, "Linking to track %s", track.trackId.string().ascii().data());
GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN(m_pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "append-pipeline-before-link");
ASSERT(!GST_PAD_IS_LINKED(track.entryPad.get()));
gst_pad_link(demuxerSrcPad, track.entryPad.get());
Expand Down
4 changes: 3 additions & 1 deletion Source/WebCore/platform/graphics/gstreamer/mse/AppendPipeline.h
View file
Expand Up @@ -98,6 +98,7 @@ class AppendPipeline {
#endif

void initializeElements(AppendPipeline*, GstBin*);
bool isLinked() const { return gst_pad_is_linked(entryPad.get()); }
};

void handleErrorSyncMessage(GstMessage*);
Expand All @@ -124,7 +125,8 @@ class AppendPipeline {
static AtomString generateTrackId(StreamType, int padIndex);
enum class CreateTrackResult { TrackCreated, TrackIgnored, AppendParsingFailed };
std::pair<CreateTrackResult, AppendPipeline::Track*> tryCreateTrackFromPad(GstPad* demuxerSrcPad, int padIndex);
AppendPipeline::Track* tryMatchPadToExistingTrack(GstPad* demuxerSrcPad);

bool recycleTrackForPad(GstPad*);
void linkPadWithTrack(GstPad* demuxerSrcPad, Track&);

void consumeAppsinksAvailableSamples();
Expand Down

0 comments on commit 296d132

Please sign in to comment.