Improved AV Reading/Writing

FFMediaToolkit/Audio/AudioData.cs

@@ -34,9 +34,51 @@ internal AudioData(AudioFrame frame)
         /// </summary>
         /// <param name="channel">The index of audio channel that should be retrieved, allowed range: [0..<see cref="NumChannels"/>).</param>
         /// <returns>The span with samples in range of [-1.0, ..., 1.0].</returns>
-        public ReadOnlySpan<float> GetChannelData(uint channel)
+        public Span<float> GetChannelData(uint channel)
         {
             return frame.GetChannelData(channel);
         }
+
+        /// <summary>
+        /// Copies raw multichannel audio data from this frame to a heap allocated array.
+        /// </summary>
+        /// <returns>
+        /// The span with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns;
+        /// samples in range of [-1.0, ..., 1.0].
+        /// </returns>
+        public float[][] GetSampleData()
+        {
+            return frame.GetSampleData();
+        }
+
+        /// <summary>
+        /// Updates the specified channel of this audio frame with the given sample data.
+        /// </summary>
+        /// <param name="samples">An array of samples with length <see cref="NumSamples"/>.</param>
+        /// <param name="channel">The index of audio channel that should be updated, allowed range: [0..<see cref="NumChannels"/>).</param>
+        public void UpdateChannelData(float[] samples, uint channel)
+        {
+            frame.UpdateChannelData(samples, channel);
+        }
+
+        /// <summary>
+        /// Updates this audio frame with the specified multi-channel sample data.
+        /// </summary>
+        /// <param name="samples">
+        /// A 2D jagged array of multi-channel sample data
+        /// with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns.
+        /// </param>
+        public void UpdateFromSampleData(float[][] samples)
+        {
+            frame.UpdateFromSampleData(samples);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged resources associated with this instance.
+        /// </summary>
+        public void Dispose()
+        {
+            frame.Dispose();
+        }
     }
 }

FFMediaToolkit/Audio/SampleFormat.cs

@@ -0,0 +1,60 @@
+namespace FFMediaToolkit.Audio
+{
+    using FFmpeg.AutoGen;
+
+    /// <summary>
+    /// Enumerates common audio sample formats supported by FFmpeg.
+    /// </summary>
+    public enum SampleFormat
+    {
+        /// <summary>
+        /// Unsupported/Unknown.
+        /// </summary>
+        None = AVSampleFormat.AV_SAMPLE_FMT_NONE,
+
+        /// <summary>
+        /// Unsigned 8-bit integer.
+        /// </summary>
+        UnsignedByte = AVSampleFormat.AV_SAMPLE_FMT_U8,
+
+        /// <summary>
+        /// Signed 16-bit integer.
+        /// </summary>
+        SignedWord = AVSampleFormat.AV_SAMPLE_FMT_S16,
+
+        /// <summary>
+        /// Signed 32-bit integer.
+        /// </summary>
+        SignedDWord = AVSampleFormat.AV_SAMPLE_FMT_S32,
+
+        /// <summary>
+        /// Single precision floating point.
+        /// </summary>
+        Single = AVSampleFormat.AV_SAMPLE_FMT_FLT,
+
+        /// <summary>
+        /// Double precision floating point.
+        /// </summary>
+        Double = AVSampleFormat.AV_SAMPLE_FMT_DBL,
+
+        /// <summary>
+        /// Signed 16-bit integer (planar).
+        /// </summary>
+        SignedWordP = AVSampleFormat.AV_SAMPLE_FMT_S16P,
+
+        /// <summary>
+        /// Signed 32-bit integer (planar).
+        /// </summary>
+        SignedDWordP = AVSampleFormat.AV_SAMPLE_FMT_S32P,
+
+        /// <summary>
+        /// Single precision floating point (planar).
+        /// </summary>
+        SingleP = AVSampleFormat.AV_SAMPLE_FMT_FLTP,
+
+        /// <summary>
+        /// Double precision floating point (planar).
+        /// </summary>
+        DoubleP = AVSampleFormat.AV_SAMPLE_FMT_DBLP,
+    }
+}

FFMediaToolkit/Common/Internal/AudioFrame.cs

@@ -1,6 +1,7 @@
 namespace FFMediaToolkit.Common.Internal
 {
     using System;
+    using FFMediaToolkit.Audio;
     using FFMediaToolkit.Helpers;
     using FFmpeg.AutoGen;
 
@@ -33,6 +34,11 @@ public AudioFrame(AVFrame* frame)
         /// </summary>
         public int NumSamples => Pointer != null ? Pointer->nb_samples : default;
 
+        /// <summary>
+        /// Gets the sample rate.
+        /// </summary>
+        public int SampleRate => Pointer != null ? Pointer->sample_rate : default;
+
         /// <summary>
         /// Gets the number of channels.
         /// </summary>
@@ -41,20 +47,31 @@ public AudioFrame(AVFrame* frame)
         /// <summary>
         /// Gets the audio sample format.
         /// </summary>
-        public AVSampleFormat SampleFormat => Pointer != null ? (AVSampleFormat)Pointer->format : AVSampleFormat.AV_SAMPLE_FMT_NONE;
+        public SampleFormat SampleFormat => Pointer != null ? (SampleFormat)Pointer->format : SampleFormat.None;
+
+        /// <summary>
+        /// Gets the channel layout.
+        /// </summary>
+        internal long ChannelLayout => Pointer != null ? (long)Pointer->channel_layout : default;
 
         /// <summary>
         /// Creates an audio frame with given dimensions and allocates a buffer for it.
         /// </summary>
-        /// <param name="num_samples">The number of samples in the audio frame.</param>
+        /// <param name="sample_rate">The sample rate of the audio frame.</param>
         /// <param name="num_channels">The number of channels in the audio frame.</param>
+        /// <param name="num_samples">The number of samples in the audio frame.</param>
+        /// <param name="channel_layout">The channel layout to be used by the audio frame.</param>
         /// <param name="sampleFormat">The audio sample format.</param>
         /// <returns>The new audio frame.</returns>
-        public static AudioFrame Create(int num_samples, int num_channels, AVSampleFormat sampleFormat)
+        public static AudioFrame Create(int sample_rate, int num_channels, int num_samples, long channel_layout, SampleFormat sampleFormat)
         {
             var frame = ffmpeg.av_frame_alloc();
-            frame->nb_samples = num_samples;
+
+            frame->sample_rate = sample_rate;
             frame->channels = num_channels;
+
+            frame->nb_samples = num_samples;
+            frame->channel_layout = (ulong)channel_layout;
             frame->format = (int)sampleFormat;
 
             ffmpeg.av_frame_get_buffer(frame, 32);
@@ -73,9 +90,94 @@ public static AudioFrame Create(int num_samples, int num_channels, AVSampleForma
         /// </summary>
         /// <param name="channel">The index of audio channel that should be retrieved, allowed range: [0..<see cref="NumChannels"/>).</param>
         /// <returns>The span with samples in range of [-1.0, ..., 1.0].</returns>
-        public ReadOnlySpan<float> GetChannelData(uint channel)
+        public Span<float> GetChannelData(uint channel)
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot extract channel data from an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+            return new Span<float>(Pointer->data[channel], NumSamples);
+        }
+
+        /// <summary>
+        /// Copies raw multichannel audio data from this frame to a heap allocated array.
+        /// </summary>
+        /// <returns>
+        /// The span with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns;
+        /// samples in range of [-1.0, ..., 1.0].
+        /// </returns>
+        public float[][] GetSampleData()
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot extract sample data from an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            var samples = new float[NumChannels][];
+
+            for (uint ch = 0; ch < NumChannels; ch++)
+            {
+                samples[ch] = new float[NumSamples];
+
+                var channelData = GetChannelData(ch);
+                var sampleData = new Span<float>(samples[ch], 0, NumSamples);
+
+                channelData.CopyTo(sampleData);
+            }
+
+            return samples;
+        }
+
+        /// <summary>
+        /// Updates the specified channel of this audio frame with the given sample data.
+        /// </summary>
+        /// <param name="samples">An array of samples with length <see cref="NumSamples"/>.</param>
+        /// <param name="channel">The index of audio channel that should be updated, allowed range: [0..<see cref="NumChannels"/>).</param>
+        public void UpdateChannelData(float[] samples, uint channel)
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot update channel data of an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            var frameData = GetChannelData(channel);
+            var sampleData = new Span<float>(samples, 0, NumSamples);
+
+            sampleData.CopyTo(frameData);
+        }
+
+        /// <summary>
+        /// Updates this audio frame with the specified multi-channel sample data.
+        /// </summary>
+        /// <param name="samples">
+        /// A 2D jagged array of multi-channel sample data
+        /// with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns.
+        /// </param>
+        public void UpdateFromSampleData(float[][] samples)
         {
-            return new ReadOnlySpan<float>(Pointer->data[channel], NumSamples);
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot update sample data of an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            for (uint ch = 0; ch < NumChannels; ch++)
+            {
+                var newData = new Span<float>(samples[ch], 0, NumSamples);
+                var frameData = GetChannelData(ch);
+                newData.CopyTo(frameData);
+            }
+        }
+
+        /// <summary>
+        /// Updates this audio frame with the specified audio data.
+        /// (<see cref="AudioData.NumSamples"/> and <see cref="AudioData.NumChannels"/>
+        /// should match the respective values for this instance!).
+        /// </summary>
+        /// <param name="audioData">The audio data.</param>
+        public void UpdateFromAudioData(AudioData audioData)
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot update data of an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            for (uint ch = 0; ch < NumChannels; ch++)
+            {
+                var newData = audioData.GetChannelData(ch);
+                var currData = GetChannelData(ch);
+
+                newData.CopyTo(currData);
+            }
         }
 
         /// <inheritdoc/>

FFMediaToolkit/Decoding/AudioStream.cs

@@ -5,12 +5,15 @@
     using FFMediaToolkit.Audio;
     using FFMediaToolkit.Common.Internal;
     using FFMediaToolkit.Decoding.Internal;
+    using FFmpeg.AutoGen;
 
     /// <summary>
     /// Represents an audio stream in the <see cref="MediaFile"/>.
     /// </summary>
-    public class AudioStream : MediaStream
+    public unsafe class AudioStream : MediaStream
     {
+        private SwrContext* swrContext;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="AudioStream"/> class.
         /// </summary>
@@ -19,6 +22,18 @@ public class AudioStream : MediaStream
         internal AudioStream(Decoder stream, MediaOptions options)
             : base(stream, options)
         {
+            swrContext = ffmpeg.swr_alloc_set_opts(
+                null,
+                Info.ChannelLayout,
+                (AVSampleFormat)SampleFormat.SingleP,
+                Info.SampleRate,
+                Info.ChannelLayout,
+                (AVSampleFormat)Info.SampleFormat,
+                Info.SampleRate,
+                0,
+                null);
+
+            ffmpeg.swr_init(swrContext);
         }
 
         /// <summary>
@@ -33,7 +48,17 @@ internal AudioStream(Decoder stream, MediaOptions options)
         public new AudioData GetNextFrame()
         {
             var frame = base.GetNextFrame() as AudioFrame;
-            return new AudioData(frame);
+
+            var converted = AudioFrame.Create(
+                frame.SampleRate,
+                frame.NumChannels,
+                frame.NumSamples,
+                frame.ChannelLayout,
+                SampleFormat.SingleP);
+
+            ffmpeg.swr_convert_frame(swrContext, converted.Pointer, frame.Pointer);
+
+            return new AudioData(converted);
         }
 
         /// <summary>
@@ -89,5 +114,16 @@ public bool TryGetFrame(TimeSpan time, out AudioData data)
                 return false;
             }
         }
+
+        /// <inheritdoc/>
+        public override void Dispose()
+        {
+            fixed (SwrContext** ptr = &swrContext)
+            {
+                ffmpeg.swr_free(ptr);
+            }
+
+            base.Dispose();
+        }
     }
 }

FFMediaToolkit/Decoding/AudioStreamInfo.cs

@@ -1,9 +1,8 @@
 namespace FFMediaToolkit.Decoding
 {
-    using System;
+    using FFMediaToolkit.Audio;
     using FFMediaToolkit.Common;
     using FFMediaToolkit.Decoding.Internal;
-    using FFMediaToolkit.Helpers;
     using FFmpeg.AutoGen;
 
     /// <summary>
@@ -22,10 +21,9 @@ internal unsafe AudioStreamInfo(AVStream* stream, InputContainer container)
             var codec = stream->codec;
             NumChannels = codec->channels;
             SampleRate = codec->sample_rate;
-            long num_samples = stream->duration >= 0 ? stream->duration : container.Pointer->duration;
-            AvgNumSamplesPerFrame = (int)Math.Round((double)num_samples / NumberOfFrames.Value);
-            SampleFormat = codec->sample_fmt.FormatEnum(14);
-            AvSampleFormat = codec->sample_fmt;
+            SamplesPerFrame = codec->frame_size > 0 ? codec->frame_size : codec->sample_rate / 20;
+            SampleFormat = (SampleFormat)codec->sample_fmt;
+            ChannelLayout = ffmpeg.av_get_default_channel_layout(codec->channels);
         }
 
         /// <summary>
@@ -42,16 +40,16 @@ internal unsafe AudioStreamInfo(AVStream* stream, InputContainer container)
         /// Gets the average number of samples per frame (chunk of samples) calculated from metadata.
         /// It is used to calculate timestamps in the internal decoder methods.
         /// </summary>
-        public int AvgNumSamplesPerFrame { get; }
+        public int SamplesPerFrame { get; }
 
         /// <summary>
-        /// Gets a lowercase string representing the audio sample format.
+        /// Gets the audio sample format.
         /// </summary>
-        public string SampleFormat { get; }
+        public SampleFormat SampleFormat { get; }
 
         /// <summary>
-        /// Gets the audio sample format.
+        /// Gets the channel layout for this stream.
         /// </summary>
-        internal AVSampleFormat AvSampleFormat { get; }
+        internal long ChannelLayout { get; }
     }
 }

FFMediaToolkit/Decoding/Internal/InputContainer.cs

@@ -152,9 +152,7 @@ private void OpenStreams(MediaOptions options)
             for (int i = 0; i < Pointer->nb_streams; i++)
             {
                 var stream = Pointer->streams[i];
-                var mode = (MediaMode)(1 << (int)stream->codec->codec_type);
-
-                if (!options.StreamsToLoad.HasFlag(mode))
+                if (!options.ShouldLoadStreamsOfType(stream->codec->codec_type))
                     continue;
 
                 try