From b15552b5510df14cdc801e84b031ebc164133b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Per=C5=BCy=C5=82o?= Date: Wed, 24 May 2023 15:05:33 +0200 Subject: [PATCH 1/3] Add support for remote stream's decoder configuration parsing --- lib/membrane_h264_plugin/parser.ex | 37 +++++++++--- .../parser/decoder_configuration_record.ex | 56 +++++++++++++++++++ 2 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 lib/membrane_h264_plugin/parser/decoder_configuration_record.ex diff --git a/lib/membrane_h264_plugin/parser.ex b/lib/membrane_h264_plugin/parser.ex index df8c360..b4d1fcc 100644 --- a/lib/membrane_h264_plugin/parser.ex +++ b/lib/membrane_h264_plugin/parser.ex @@ -89,7 +89,8 @@ defmodule Membrane.H264.Parser do profile: nil, previous_timestamps: {nil, nil}, framerate: opts.framerate, - au_counter: 0 + au_counter: 0, + frame_prefix: <<>> } {[], state} @@ -97,20 +98,42 @@ defmodule Membrane.H264.Parser do @impl true def handle_stream_format(:input, stream_format, _ctx, state) do - mode = + state = case stream_format do - %RemoteStream{type: :bytestream} -> :bytestream - %H264.RemoteStream{alignment: :nalu} -> :nalu_aligned - %H264.RemoteStream{alignment: :au} -> :au_aligned + %RemoteStream{type: :bytestream} -> + %{state | mode: :bytestream} + + %H264.RemoteStream{alignment: alignment, decoder_configuration_record: dcr} -> + mode = + case alignment do + :nalu -> :nalu_aligned + :au -> :au_aligned + end + + frame_prefix = + if dcr do + {:ok, %{sps: sps, pps: pps}} = __MODULE__.DecoderConfigurationRecord.parse(dcr) + + Enum.concat([[<<>>], sps, pps]) |> Enum.join(<<0, 0, 1>>) + else + <<>> + end + + %{state | mode: mode, frame_prefix: frame_prefix} end - state = %{state | mode: mode} {[], state} end @impl true def handle_process(:input, %Membrane.Buffer{} = buffer, _ctx, state) do - {nalus_payloads_list, nalu_splitter} = NALuSplitter.split(buffer.payload, state.nalu_splitter) + {payload, state} = + case state.frame_prefix do + <<>> -> {buffer.payload, state} + prefix -> {prefix <> buffer.payload, %{state | frame_prefix: <<>>}} + end + + {nalus_payloads_list, nalu_splitter} = NALuSplitter.split(payload, state.nalu_splitter) {nalus_payloads_list, nalu_splitter} = if state.mode != :bytestream do diff --git a/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex b/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex new file mode 100644 index 0000000..007219b --- /dev/null +++ b/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex @@ -0,0 +1,56 @@ +defmodule Membrane.H264.Parser.DecoderConfigurationRecord do + @moduledoc """ + Utility functions for parsing AVC Configuration Record + """ + @enforce_keys [ + :sps, + :pps, + :avc_profile_indication, + :avc_level, + :profile_compatibility, + :length_size + ] + defstruct @enforce_keys + + @type t() :: %__MODULE__{ + sps: [binary()], + pps: [binary()], + avc_profile_indication: non_neg_integer(), + profile_compatibility: non_neg_integer(), + avc_level: non_neg_integer(), + length_size: non_neg_integer() + } + + @spec parse(binary()) :: {:ok, t()} | {:error, any()} + def parse( + <<1::8, avc_profile_indication::8, profile_compatibility::8, avc_level::8, 0b111111::6, + length_size::2, 0b111::3, rest::bitstring>> + ) do + {sps, rest} = parse_sps(rest) + {pps, _rest} = parse_pps(rest) + + %__MODULE__{ + sps: sps, + pps: pps, + avc_profile_indication: avc_profile_indication, + profile_compatibility: profile_compatibility, + avc_level: avc_level, + length_size: length_size + } + |> then(&{:ok, &1}) + end + + def parse(_data), do: {:error, :unknown_pattern} + + defp parse_sps(<>) do + do_parse_array(num_of_sps, rest) + end + + defp parse_pps(<>), do: do_parse_array(num_of_pps, rest) + + defp do_parse_array(amount, rest, acc \\ []) + defp do_parse_array(0, rest, acc), do: {Enum.reverse(acc), rest} + + defp do_parse_array(remaining, <>, acc), + do: do_parse_array(remaining - 1, rest, [data | acc]) +end From 6014df0c11562265acd192ecba28b64e9c4e3d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Per=C5=BCy=C5=82o?= Date: Tue, 30 May 2023 16:40:08 +0200 Subject: [PATCH 2/3] Adjust to CR --- lib/membrane_h264_plugin/parser.ex | 11 +++++++++-- .../parser/decoder_configuration_record.ex | 16 +++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/membrane_h264_plugin/parser.ex b/lib/membrane_h264_plugin/parser.ex index b4d1fcc..bbe876f 100644 --- a/lib/membrane_h264_plugin/parser.ex +++ b/lib/membrane_h264_plugin/parser.ex @@ -40,6 +40,8 @@ defmodule Membrane.H264.Parser do alias Membrane.{Buffer, H264, RemoteStream} alias Membrane.H264.Parser.{AUSplitter, Format, NALuParser, NALuSplitter} + alias __MODULE__.DecoderConfigurationRecord + def_input_pad :input, demand_unit: :buffers, demand_mode: :auto, @@ -90,7 +92,8 @@ defmodule Membrane.H264.Parser do previous_timestamps: {nil, nil}, framerate: opts.framerate, au_counter: 0, - frame_prefix: <<>> + frame_prefix: <<>>, + parameter_sets_present?: byte_size(opts.sps) > 0 or byte_size(opts.pps) > 0 } {[], state} @@ -112,7 +115,11 @@ defmodule Membrane.H264.Parser do frame_prefix = if dcr do - {:ok, %{sps: sps, pps: pps}} = __MODULE__.DecoderConfigurationRecord.parse(dcr) + if state.parameter_sets_present? do + raise "Parameter sets were already provided as the options to the parser and parameter sets from the decoder configuration record could overwrite them." + end + + {:ok, %{sps: sps, pps: pps}} = DecoderConfigurationRecord.parse(dcr) Enum.concat([[<<>>], sps, pps]) |> Enum.join(<<0, 0, 1>>) else diff --git a/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex b/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex index 007219b..74de395 100644 --- a/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex +++ b/lib/membrane_h264_plugin/parser/decoder_configuration_record.ex @@ -1,6 +1,8 @@ defmodule Membrane.H264.Parser.DecoderConfigurationRecord do @moduledoc """ - Utility functions for parsing AVC Configuration Record + Utility functions for parsing AVC Configuration Record. + + The structure of the record is described in section 5.2.4.1.1 of MPEG-4 part 15 (ISO/IEC 14496-15). """ @enforce_keys [ :sps, @@ -8,23 +10,27 @@ defmodule Membrane.H264.Parser.DecoderConfigurationRecord do :avc_profile_indication, :avc_level, :profile_compatibility, - :length_size + :length_size_minus_one ] defstruct @enforce_keys + @typedoc "Structure representing the Decoder Configuartion Record" @type t() :: %__MODULE__{ sps: [binary()], pps: [binary()], avc_profile_indication: non_neg_integer(), profile_compatibility: non_neg_integer(), avc_level: non_neg_integer(), - length_size: non_neg_integer() + length_size_minus_one: non_neg_integer() } + @doc """ + Parses the DCR. + """ @spec parse(binary()) :: {:ok, t()} | {:error, any()} def parse( <<1::8, avc_profile_indication::8, profile_compatibility::8, avc_level::8, 0b111111::6, - length_size::2, 0b111::3, rest::bitstring>> + length_size_minus_one::2, 0b111::3, rest::bitstring>> ) do {sps, rest} = parse_sps(rest) {pps, _rest} = parse_pps(rest) @@ -35,7 +41,7 @@ defmodule Membrane.H264.Parser.DecoderConfigurationRecord do avc_profile_indication: avc_profile_indication, profile_compatibility: profile_compatibility, avc_level: avc_level, - length_size: length_size + length_size_minus_one: length_size_minus_one } |> then(&{:ok, &1}) end From 236fbf389a83ccc9fdecd615c69ef74cad28d4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Per=C5=BCy=C5=82o?= Date: Thu, 1 Jun 2023 15:43:49 +0200 Subject: [PATCH 3/3] Bump version to 0.3.0 --- README.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8c1f6fd..22731cc 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The package can be installed by adding `membrane_h264_plugin` to your list of de ```elixir def deps do [ - {:membrane_h264_plugin, "~> 0.2.0"} + {:membrane_h264_plugin, "~> 0.3.0"} ] end ``` diff --git a/mix.exs b/mix.exs index ef2006d..d11a0fd 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Membrane.H264.TODO.Mixfile do use Mix.Project - @version "0.2.0" + @version "0.3.0" @github_url "https://github.com/membraneframework-labs/membrane_h264_plugin" def project do