Merge pull request #57 from yodadude2003/develop

Improved AV Reading/Writing

FFMediaToolkit/Audio/AudioData.cs

@@ -34,9 +34,51 @@ internal AudioData(AudioFrame frame)
         /// </summary>
         /// <param name="channel">The index of audio channel that should be retrieved, allowed range: [0..<see cref="NumChannels"/>).</param>
         /// <returns>The span with samples in range of [-1.0, ..., 1.0].</returns>
-        public ReadOnlySpan<float> GetChannelData(uint channel)
+        public Span<float> GetChannelData(uint channel)
         {
             return frame.GetChannelData(channel);
         }
+
+        /// <summary>
+        /// Copies raw multichannel audio data from this frame to a heap allocated array.
+        /// </summary>
+        /// <returns>
+        /// The span with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns;
+        /// samples in range of [-1.0, ..., 1.0].
+        /// </returns>
+        public float[][] GetSampleData()
+        {
+            return frame.GetSampleData();
+        }
+
+        /// <summary>
+        /// Updates the specified channel of this audio frame with the given sample data.
+        /// </summary>
+        /// <param name="samples">An array of samples with length <see cref="NumSamples"/>.</param>
+        /// <param name="channel">The index of audio channel that should be updated, allowed range: [0..<see cref="NumChannels"/>).</param>
+        public void UpdateChannelData(float[] samples, uint channel)
+        {
+            frame.UpdateChannelData(samples, channel);
+        }
+
+        /// <summary>
+        /// Updates this audio frame with the specified multi-channel sample data.
+        /// </summary>
+        /// <param name="samples">
+        /// A 2D jagged array of multi-channel sample data
+        /// with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns.
+        /// </param>
+        public void UpdateFromSampleData(float[][] samples)
+        {
+            frame.UpdateFromSampleData(samples);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged resources associated with this instance.
+        /// </summary>
+        public void Dispose()
+        {
+            frame.Dispose();
+        }
     }
 }

FFMediaToolkit/Audio/SampleFormat.cs

@@ -0,0 +1,60 @@
+namespace FFMediaToolkit.Audio
+{
+    using FFmpeg.AutoGen;
+
+    /// <summary>
+    /// Enumerates common audio sample formats supported by FFmpeg.
+    /// </summary>
+    public enum SampleFormat
+    {
+        /// <summary>
+        /// Unsupported/Unknown.
+        /// </summary>
+        None = AVSampleFormat.AV_SAMPLE_FMT_NONE,
+
+        /// <summary>
+        /// Unsigned 8-bit integer.
+        /// </summary>
+        UnsignedByte = AVSampleFormat.AV_SAMPLE_FMT_U8,
+
+        /// <summary>
+        /// Signed 16-bit integer.
+        /// </summary>
+        SignedWord = AVSampleFormat.AV_SAMPLE_FMT_S16,
+
+        /// <summary>
+        /// Signed 32-bit integer.
+        /// </summary>
+        SignedDWord = AVSampleFormat.AV_SAMPLE_FMT_S32,
+
+        /// <summary>
+        /// Single precision floating point.
+        /// </summary>
+        Single = AVSampleFormat.AV_SAMPLE_FMT_FLT,
+
+        /// <summary>
+        /// Double precision floating point.
+        /// </summary>
+        Double = AVSampleFormat.AV_SAMPLE_FMT_DBL,
+
+        /// <summary>
+        /// Signed 16-bit integer (planar).
+        /// </summary>
+        SignedWordP = AVSampleFormat.AV_SAMPLE_FMT_S16P,
+
+        /// <summary>
+        /// Signed 32-bit integer (planar).
+        /// </summary>
+        SignedDWordP = AVSampleFormat.AV_SAMPLE_FMT_S32P,
+
+        /// <summary>
+        /// Single precision floating point (planar).
+        /// </summary>
+        SingleP = AVSampleFormat.AV_SAMPLE_FMT_FLTP,
+
+        /// <summary>
+        /// Double precision floating point (planar).
+        /// </summary>
+        DoubleP = AVSampleFormat.AV_SAMPLE_FMT_DBLP,
+    }
+}

FFMediaToolkit/Common/ContainerMetadata.cs

@@ -1,7 +1,7 @@
 namespace FFMediaToolkit.Common
 {
-    using FFmpeg.AutoGen;
     using System.Collections.Generic;
+    using FFmpeg.AutoGen;
 
     /// <summary>
     /// Represents multimedia file metadata info.

FFMediaToolkit/Common/Internal/AudioFrame.cs

@@ -1,5 +1,7 @@
-namespace FFMediaToolkit.Common.Internal {
+namespace FFMediaToolkit.Common.Internal
+{
     using System;
+    using FFMediaToolkit.Audio;
     using FFMediaToolkit.Helpers;
     using FFmpeg.AutoGen;
 
@@ -32,6 +34,11 @@ public AudioFrame(AVFrame* frame)
         /// </summary>
         public int NumSamples => Pointer != null ? Pointer->nb_samples : default;
 
+        /// <summary>
+        /// Gets the sample rate.
+        /// </summary>
+        public int SampleRate => Pointer != null ? Pointer->sample_rate : default;
+
         /// <summary>
         /// Gets the number of channels.
         /// </summary>
@@ -40,20 +47,31 @@ public AudioFrame(AVFrame* frame)
         /// <summary>
         /// Gets the audio sample format.
         /// </summary>
-        public AVSampleFormat SampleFormat => Pointer != null ? (AVSampleFormat) Pointer->format : AVSampleFormat.AV_SAMPLE_FMT_NONE;
+        public SampleFormat SampleFormat => Pointer != null ? (SampleFormat)Pointer->format : SampleFormat.None;
+
+        /// <summary>
+        /// Gets the channel layout.
+        /// </summary>
+        internal long ChannelLayout => Pointer != null ? (long)Pointer->channel_layout : default;
 
         /// <summary>
         /// Creates an audio frame with given dimensions and allocates a buffer for it.
         /// </summary>
-        /// <param name="num_samples">The number of samples in the audio frame.</param>
+        /// <param name="sample_rate">The sample rate of the audio frame.</param>
         /// <param name="num_channels">The number of channels in the audio frame.</param>
+        /// <param name="num_samples">The number of samples in the audio frame.</param>
+        /// <param name="channel_layout">The channel layout to be used by the audio frame.</param>
         /// <param name="sampleFormat">The audio sample format.</param>
         /// <returns>The new audio frame.</returns>
-        public static AudioFrame Create(int num_samples, int num_channels, AVSampleFormat sampleFormat)
+        public static AudioFrame Create(int sample_rate, int num_channels, int num_samples, long channel_layout, SampleFormat sampleFormat)
         {
             var frame = ffmpeg.av_frame_alloc();
-            frame->nb_samples = num_samples;
+
+            frame->sample_rate = sample_rate;
             frame->channels = num_channels;
+
+            frame->nb_samples = num_samples;
+            frame->channel_layout = (ulong)channel_layout;
             frame->format = (int)sampleFormat;
 
             ffmpeg.av_frame_get_buffer(frame, 32);
@@ -72,9 +90,94 @@ public static AudioFrame Create(int num_samples, int num_channels, AVSampleForma
         /// </summary>
         /// <param name="channel">The index of audio channel that should be retrieved, allowed range: [0..<see cref="NumChannels"/>).</param>
         /// <returns>The span with samples in range of [-1.0, ..., 1.0].</returns>
-        public ReadOnlySpan<float> GetChannelData(uint channel)
+        public Span<float> GetChannelData(uint channel)
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot extract channel data from an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+            return new Span<float>(Pointer->data[channel], NumSamples);
+        }
+
+        /// <summary>
+        /// Copies raw multichannel audio data from this frame to a heap allocated array.
+        /// </summary>
+        /// <returns>
+        /// The span with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns;
+        /// samples in range of [-1.0, ..., 1.0].
+        /// </returns>
+        public float[][] GetSampleData()
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot extract sample data from an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            var samples = new float[NumChannels][];
+
+            for (uint ch = 0; ch < NumChannels; ch++)
+            {
+                samples[ch] = new float[NumSamples];
+
+                var channelData = GetChannelData(ch);
+                var sampleData = new Span<float>(samples[ch], 0, NumSamples);
+
+                channelData.CopyTo(sampleData);
+            }
+
+            return samples;
+        }
+
+        /// <summary>
+        /// Updates the specified channel of this audio frame with the given sample data.
+        /// </summary>
+        /// <param name="samples">An array of samples with length <see cref="NumSamples"/>.</param>
+        /// <param name="channel">The index of audio channel that should be updated, allowed range: [0..<see cref="NumChannels"/>).</param>
+        public void UpdateChannelData(float[] samples, uint channel)
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot update channel data of an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            var frameData = GetChannelData(channel);
+            var sampleData = new Span<float>(samples, 0, NumSamples);
+
+            sampleData.CopyTo(frameData);
+        }
+
+        /// <summary>
+        /// Updates this audio frame with the specified multi-channel sample data.
+        /// </summary>
+        /// <param name="samples">
+        /// A 2D jagged array of multi-channel sample data
+        /// with <see cref="NumChannels"/> rows and <see cref="NumSamples"/> columns.
+        /// </param>
+        public void UpdateFromSampleData(float[][] samples)
         {
-            return new ReadOnlySpan<float>(Pointer->data[channel], NumSamples);
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot update sample data of an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            for (uint ch = 0; ch < NumChannels; ch++)
+            {
+                var newData = new Span<float>(samples[ch], 0, NumSamples);
+                var frameData = GetChannelData(ch);
+                newData.CopyTo(frameData);
+            }
+        }
+
+        /// <summary>
+        /// Updates this audio frame with the specified audio data.
+        /// (<see cref="AudioData.NumSamples"/> and <see cref="AudioData.NumChannels"/>
+        /// should match the respective values for this instance!).
+        /// </summary>
+        /// <param name="audioData">The audio data.</param>
+        public void UpdateFromAudioData(AudioData audioData)
+        {
+            if (SampleFormat != SampleFormat.SingleP)
+                throw new Exception("Cannot update data of an AudioFrame with a SampleFormat not equal to SampleFormat.SingleP");
+
+            for (uint ch = 0; ch < NumChannels; ch++)
+            {
+                var newData = audioData.GetChannelData(ch);
+                var currData = GetChannelData(ch);
+
+                newData.CopyTo(currData);
+            }
         }
 
         /// <inheritdoc/>

FFMediaToolkit/Common/Internal/ImageConverter.cs

@@ -58,14 +58,12 @@ internal void FillAVFrame(ImageData bitmap, VideoFrame destinationFrame)
         /// </summary>
         /// <param name="videoFrame">The video frame to convert.</param>
         /// <param name="destination">The destination <see cref="ImageData"/>.</param>
-        internal void AVFrameToBitmap(VideoFrame videoFrame, ImageData destination)
+        /// <param name="stride">Size of the single bitmap row.</param>
+        internal void AVFrameToBitmap(VideoFrame videoFrame, byte* destination, int stride)
         {
-            fixed (byte* ptr = destination.Data)
-            {
-                var data = new byte*[4] { ptr, null, null, null };
-                var linesize = new int[4] { destination.Stride, 0, 0, 0 };
-                ffmpeg.sws_scale(Pointer, videoFrame.Pointer->data, videoFrame.Pointer->linesize, 0, videoFrame.Layout.Height, data, linesize);
-            }
+            var data = new byte*[4] { destination, null, null, null };
+            var linesize = new int[4] { stride, 0, 0, 0 };
+            ffmpeg.sws_scale(Pointer, videoFrame.Pointer->data, videoFrame.Pointer->linesize, 0, videoFrame.Layout.Height, data, linesize);
         }
 
         /// <inheritdoc/>

FFMediaToolkit/Common/Internal/MediaPacket.cs

@@ -48,12 +48,11 @@ public int StreamIndex
         /// <summary>
         /// Allocates a new empty packet.
         /// </summary>
-        /// <param name="streamIndex">The packet stream index.</param>
         /// <returns>The new <see cref="MediaPacket"/>.</returns>
-        public static MediaPacket AllocateEmpty(int streamIndex)
+        public static MediaPacket AllocateEmpty()
         {
             var packet = ffmpeg.av_packet_alloc();
-            packet->stream_index = streamIndex;
+            packet->stream_index = -1;
             return new MediaPacket(packet);
         }
 

FFMediaToolkit/Common/Internal/VideoFrame.cs

@@ -77,11 +77,17 @@ public static VideoFrame Create(Size dimensions, AVPixelFormat pixelFormat)
         /// </summary>
         /// <param name="converter">A <see cref="ImageConverter"/> object, used for caching the FFMpeg <see cref="SwsContext"/> when converting many frames of the same video.</param>
         /// <param name="targetFormat">The output bitmap pixel format.</param>
+        /// /// <param name="targetSize">The output bitmap size.</param>
         /// <returns>A <see cref="ImageData"/> instance containing converted bitmap data.</returns>
-        public ImageData ToBitmap(ImageConverter converter, ImagePixelFormat targetFormat)
+        public ImageData ToBitmap(ImageConverter converter, ImagePixelFormat targetFormat, Size targetSize)
         {
-            var bitmap = ImageData.CreatePooled(Layout, targetFormat);
-            converter.AVFrameToBitmap(this, bitmap);
+            var bitmap = ImageData.CreatePooled(targetSize, targetFormat); // Rents memory for the output bitmap.
+            fixed (byte* ptr = bitmap.Data)
+            {
+                // Converts the raw video frame using the given size and pixel format and writes it to the ImageData bitmap.
+                converter.AVFrameToBitmap(this, ptr, bitmap.Stride);
+            }
+
             return bitmap;
         }
 

FFMediaToolkit/Common/Internal/MediaType.cs → FFMediaToolkit/Common/MediaType.cs

@@ -1,9 +1,9 @@
-namespace FFMediaToolkit.Common.Internal
+namespace FFMediaToolkit.Common
 {
     /// <summary>
     /// Represents the multimedia stream types.
     /// </summary>
-    internal enum MediaType
+    public enum MediaType
     {
         /// <summary>
         /// Other media type not supported by the FFMediaToolkit.