Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add timestamp generation for profiles baseline, constrained baseline #21

Merged
merged 3 commits into from
Feb 13, 2023
Merged

Add timestamp generation for profiles baseline, constrained baseline #21

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
648f782
Add timestamp generation for profiles baseline, constrained baseline
sgfn Feb 9, 2023
7cbe09d
Fixes after review
sgfn Feb 10, 2023
1525e9b
Bugfix regarding timestamps in nalu_aligned mode
sgfn Feb 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 81 additions & 37 deletions lib/membrane_h264_plugin/parser.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ defmodule Membrane.H264.Parser do
* Receiving `%Membrane.H264.RemoteStream{alignment: :au}` results in the parser mode being set to `:au_aligned`

The distinguishment between parser modes was introduced to eliminate the redundant operations and to provide a reliable way
for timestamps rewritting:
* in the `:bytestream` mode, the output buffers have their `:pts` and `:dts` set to nil
for rewriting of timestamps:
* in the `:bytestream` mode:
* if option `:framerate` is set to nil, the output buffers have their `:pts` and `:dts` set to nil
* if framerate is specified, `:pts` and `:dts` will be generated automatically, based on that framerate, starting from 0
This may only be used with h264 profiles `:baseline` and `:constrained_baseline`, where `PTS==DTS`.
* in the `:nalu_aligned` mode, the output buffers have their `:pts` and `:dts` set to `:pts` and `:dts` of the
input buffer that was holding the first NAL unit making up given access unit (that is being sent inside that output buffer).
* in the `:au_aligned` mode, the output buffers have their `:pts` and `:dts` set to `:pts` and `:dts` of the input buffer
Expand Down Expand Up @@ -72,7 +75,7 @@ defmodule Membrane.H264.Parser do
description: """
Framerate of the video, represented as a tuple consisting of a numerator and the
denominator.
It's value will be sent inside the output Membrane.H264 caps.
Its value will be sent inside the output Membrane.H264 stream format.
"""
]

Expand All @@ -83,8 +86,10 @@ defmodule Membrane.H264.Parser do
nalu_parser: NALuParser.new(),
au_splitter: AUSplitter.new(),
mode: nil,
profile: nil,
previous_timestamps: {nil, nil},
framerate: opts.framerate
framerate: opts.framerate,
au_counter: 0
}

{[], state}
Expand Down Expand Up @@ -138,21 +143,7 @@ defmodule Membrane.H264.Parser do
{access_units, au_splitter}
end

{pts, dts} =
case state.mode do
:bytestream -> {nil, nil}
:nalu_aligned -> state.previous_timestamps
:au_aligned -> {buffer.pts, buffer.dts}
end

state =
if state.mode == :nalu_aligned and state.previous_timestamps != {buffer.pts, buffer.dts} do
%{state | previous_timestamps: {buffer.pts, buffer.dts}}
else
state
end

actions = prepare_actions_for_aus(access_units, pts, dts, state)
{actions, state} = prepare_actions_for_aus(access_units, state, buffer.pts, buffer.dts)

state = %{
state
Expand Down Expand Up @@ -184,13 +175,7 @@ defmodule Membrane.H264.Parser do
{remaining_nalus, au_splitter} = AUSplitter.flush(au_splitter)
maybe_improper_aus = access_units ++ [remaining_nalus]

{pts, dts} =
case state.mode do
:bytestream -> {nil, nil}
:nalu_aligned -> state.previous_timestamps
end

actions = prepare_actions_for_aus(maybe_improper_aus, pts, dts, state)
{actions, state} = prepare_actions_for_aus(maybe_improper_aus, state)
actions = if stream_format_sent?(actions, ctx), do: actions, else: []

state = %{
Expand All @@ -208,19 +193,65 @@ defmodule Membrane.H264.Parser do
{[end_of_stream: :output], state}
end

defp prepare_actions_for_aus(aus, pts, dts, state) do
Enum.reduce(aus, [], fn au, acc ->
sps_actions =
case Enum.find(au, &(&1.type == :sps)) do
nil ->
[]
defp prepare_actions_for_aus(aus, state, buffer_pts \\ nil, buffer_dts \\ nil) do
{actions, au_counter, profile} =
Enum.reduce(aus, {[], state.au_counter, state.profile}, fn au,
{actions_acc, cnt, profile} ->
{sps_actions, profile} = maybe_parse_sps(au, state, profile)
{pts, dts} = prepare_timestamps(buffer_pts, buffer_dts, state, profile, cnt)

sps_nalu ->
[stream_format: {:output, Format.from_sps(sps_nalu, framerate: state.framerate)}]
end
{actions_acc ++ sps_actions ++ [{:buffer, {:output, wrap_into_buffer(au, pts, dts)}}],
cnt + 1, profile}
end)

state = maybe_update_state(buffer_pts, buffer_dts, au_counter, profile, state)

{actions, state}
end

defp maybe_parse_sps(au, state, profile) do
case Enum.find(au, &(&1.type == :sps)) do
nil ->
{[], profile}

acc ++ sps_actions ++ [{:buffer, {:output, wrap_into_buffer(au, pts, dts)}}]
end)
sps_nalu ->
fmt = Format.from_sps(sps_nalu, framerate: state.framerate)
{[stream_format: {:output, fmt}], fmt.profile}
end
end

defp prepare_timestamps(_buffer_pts, _buffer_dts, state, profile, order_number)
sgfn marked this conversation as resolved.
Show resolved Hide resolved
when state.mode == :bytestream do
cond do
state.framerate == nil or profile == nil ->
{nil, nil}

h264_profile_tsgen_supported?(profile) ->
calculate_timestamps(state.framerate, order_number, order_number)

true ->
raise("Timestamp generation for H264 profile `#{inspect(profile)}` is unsupported")
end
end

defp prepare_timestamps(_buffer_pts, _buffer_dts, state, _profile, _order_number)
when state.mode == :nalu_aligned do
state.previous_timestamps
sgfn marked this conversation as resolved.
Show resolved Hide resolved
end

defp prepare_timestamps(buffer_pts, buffer_dts, state, _profile, _order_number)
when state.mode == :au_aligned do
{buffer_pts, buffer_dts}
end

defp maybe_update_state(buffer_pts, buffer_dts, au_counter, profile, state) do
sgfn marked this conversation as resolved.
Show resolved Hide resolved
state = %{state | profile: profile, au_counter: au_counter}

if state.mode == :nalu_aligned and state.previous_timestamps != {buffer_pts, buffer_dts} do
%{state | previous_timestamps: {buffer_pts, buffer_dts}}
else
state
end
end

defp wrap_into_buffer(access_unit, pts, dts) do
Expand Down Expand Up @@ -281,4 +312,17 @@ defmodule Membrane.H264.Parser do
do: Enum.any?(actions, &match?({:stream_format, _stream_format}, &1))

defp stream_format_sent?(_actions, _ctx), do: true

defp h264_profile_tsgen_supported?(profile),
do: profile in [:baseline, :constrained_baseline]

defp calculate_timestamps(
sgfn marked this conversation as resolved.
Show resolved Hide resolved
{frames, seconds} = _framerate,
presentation_order_number,
decoding_order_number
) do
pts = div(presentation_order_number * seconds * Membrane.Time.second(), frames)
dts = div(decoding_order_number * seconds * Membrane.Time.second(), frames)
{pts, dts}
end
end
2 changes: 1 addition & 1 deletion mix.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ defmodule Membrane.H264.TODO.Mixfile do
licenses: ["Apache-2.0"],
links: %{
"GitHub" => @github_url,
"Membrane Framework Homepage" => "https://membraneframework.org"
"Membrane Framework Homepage" => "https://membrane.stream"
}
]
end
Expand Down
Binary file added test/fixtures/input-10-720p-baseline.h264
View file
Open in desktop
Binary file not shown.
44 changes: 4 additions & 40 deletions test/integration/modes_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ defmodule Membrane.H264.ModesTest do
alias Membrane.Buffer
alias Membrane.H264.Parser
alias Membrane.H264.Parser.{AUSplitter, NALuParser, NALuSplitter}
alias Membrane.H264.Support.TestSource
alias Membrane.Testing.{Pipeline, Sink}

@h264_input_file "test/fixtures/input-10-720p.h264"
Expand Down Expand Up @@ -54,43 +55,6 @@ defmodule Membrane.H264.ModesTest do
|> elem(0)
end

defmodule ModeTestSource do
use Membrane.Source

def_options mode: []

def_output_pad :output,
demand_mode: :auto,
mode: :push,
accepted_format:
any_of(
%Membrane.RemoteStream{type: :bytestream},
%Membrane.H264.RemoteStream{alignment: alignment} when alignment in [:au, :nalu]
)

@impl true
def handle_init(_ctx, opts) do
{[], %{mode: opts.mode}}
end

@impl true
def handle_parent_notification(actions, _ctx, state) do
{actions, state}
end

@impl true
def handle_playing(_ctx, state) do
stream_format =
case state.mode do
:bytestream -> %Membrane.RemoteStream{type: :bytestream}
:nalu_aligned -> %Membrane.H264.RemoteStream{alignment: :nalu}
:au_aligned -> %Membrane.H264.RemoteStream{alignment: :au}
end

{[stream_format: {:output, stream_format}], state}
end
end

test "if the pts and dts are set to nil in :bytestream mode" do
binary = File.read!(@h264_input_file)
mode = :bytestream
Expand All @@ -99,7 +63,7 @@ defmodule Membrane.H264.ModesTest do
{:ok, _supervisor_pid, pid} =
Pipeline.start_supervised(
structure: [
child(:source, %ModeTestSource{mode: mode})
child(:source, %TestSource{mode: mode})
|> child(:parser, Parser)
|> child(:sink, Sink)
]
Expand Down Expand Up @@ -127,7 +91,7 @@ defmodule Membrane.H264.ModesTest do
{:ok, _supervisor_pid, pid} =
Pipeline.start_supervised(
structure: [
child(:source, %ModeTestSource{mode: mode})
child(:source, %TestSource{mode: mode})
|> child(:parser, Parser)
|> child(:sink, Sink)
]
Expand Down Expand Up @@ -157,7 +121,7 @@ defmodule Membrane.H264.ModesTest do
{:ok, _supervisor_pid, pid} =
Pipeline.start_supervised(
structure: [
child(:source, %ModeTestSource{mode: mode})
child(:source, %TestSource{mode: mode})
|> child(:parser, Parser)
|> child(:sink, Sink)
]
Expand Down
Loading