Audio: use a tee and an appsink (#19)

Cargo.toml

@@ -10,10 +10,12 @@ glib = { git = "https://github.com/gtk-rs/glib.git" }
 chrono = "^0"
 url = "^1"
 image = "^0"
-byteorder = "^1"
+byte-slice-cast = "^0"
 
-#gstreamer = { git = "https://github.com/sdroege/gstreamer-rs.git", features = ["v1_10"] }
-#gstreamer-audio = { git = "https://github.com/sdroege/gstreamer-rs.git", features = ["v1_10"] }
+gstreamer = { git = "https://github.com/sdroege/gstreamer-rs.git", features = ["v1_10"] }
+gstreamer-audio = { git = "https://github.com/sdroege/gstreamer-rs.git", features = ["v1_10"] }
+gstreamer-app = { git = "https://github.com/sdroege/gstreamer-rs.git", features = ["v1_10"] }
 
-gstreamer = { path = "../gstreamer-rs/gstreamer", features = ["v1_10"] }
-gstreamer-audio = { path = "../gstreamer-rs/gstreamer-audio", features = ["v1_10"] }
+#gstreamer = { path = "../gstreamer-rs/gstreamer", features = ["v1_10"] }
+#gstreamer-audio = { path = "../gstreamer-rs/gstreamer-audio", features = ["v1_10"] }
+#gstreamer-app = { path = "../gstreamer-rs/gstreamer-app", features = ["v1_10"] }

src/main.rs

@@ -5,12 +5,14 @@ extern crate cairo;
 
 extern crate gstreamer;
 extern crate gstreamer_audio;
+extern crate gstreamer_app as gst_app;
+
 extern crate glib;
 
 extern crate chrono;
 extern crate url;
 extern crate image;
-extern crate byteorder;
+extern crate byte_slice_cast;
 
 use gtk::Builder;
 

src/media/audio_buffer.rs

@@ -1,42 +1,10 @@
-extern crate byteorder;
-use byteorder::{LittleEndian, ReadBytesExt};
-
 extern crate gstreamer as gst;
-use gstreamer::PadExt;
-
-extern crate gstreamer_audio as gst_audio;
-use gstreamer_audio::AudioFormat;
-
-use std::io::Cursor;
+extern crate gstreamer_app as gst_app;
 
-use std::ops::Deref;
+extern crate byte_slice_cast;
+use byte_slice_cast::AsSliceOf;
 
-pub struct AudioInfo {
-    pub info: gst_audio::AudioInfo,
-    pub sample_duration: u64,
-}
-
-impl AudioInfo {
-    pub fn from_sink_pad(sink_pad: &gst::Pad) -> Self {
-        let caps = sink_pad.get_current_caps()
-            .expect("Couldn't get caps for audio stream");
-        let gst_audio_info = gst_audio::AudioInfo::from_caps(&caps)
-            .expect("Couldn't get audio info for audio stream");
-
-        AudioInfo {
-            sample_duration: 1_000_000_000 / (gst_audio_info.rate() as u64),
-            info: gst_audio_info,
-        }
-    }
-}
-
-impl Deref for AudioInfo {
-	type Target = gst_audio::AudioInfo;
-
-	fn deref(&self) -> &Self::Target {
-		&self.info
-	}
-}
+use std::i16;
 
 pub struct AudioBuffer {
     pub sample_duration: u64,
@@ -46,64 +14,86 @@ pub struct AudioBuffer {
 }
 
 impl AudioBuffer {
-    pub fn from_gst_buffer(info: &AudioInfo, buffer: &gst::Buffer) -> Self {
-        let duration = buffer.get_duration();
-        let sample_nb = (duration / info.sample_duration) as u32;
+    pub fn from_gst_buffer(caps: &gst::Caps, buffer: &gst::Buffer) -> Self {
+        let structure = caps.get_structure(0)
+            .expect("Couldn't get structure from audio sample");
+        let rate = structure.get::<i32>("rate")
+            .expect("Couldn't get rate from audio sample");
+        // FIXME: channels is set in appsink
+        let channels = structure.get::<i32>("channels")
+            .expect("Couldn't get channels from audio sample")
+            as usize;
+        let channels_f = channels as f64;
+
+        // assert_eq!(format, S16);
+
+        let sample_duration = 1_000_000_000 / (rate as u64);
+
+        let map = buffer.map_readable().unwrap();
+        let data = map.as_slice().as_slice_of::<i16>()
+            .expect("Couldn't get audio samples as i16");
+        let sample_nb = data.len() / channels;
+
         let mut this = AudioBuffer {
-            sample_duration: info.sample_duration,
+            sample_duration: sample_duration,
             pts: buffer.get_pts(),
-            duration: duration,
-            samples: Vec::with_capacity((info.channels() * sample_nb) as usize),
+            duration: buffer.get_duration(),
+            samples: Vec::with_capacity(sample_nb),
         };
 
-        assert_eq!(info.layout(), gst_audio::AudioLayout::Interleaved);
+        for index in 0..sample_nb {
+            let mut mono_sample = 0f64;
+            // FIXME: downmix in the pipeline (maybe done by appsink)
+            for channel in 0..channels {
+                mono_sample += data[index + channel] as f64;
+            }
+
+            this.samples.push(1f64 - (mono_sample / (i16::MAX as f64) / channels_f));
+        }
+
+        this
+    }
+
+    pub fn from_gst_sample(sample: gst::Sample) -> Self {
+        let caps = sample.get_caps()
+            .expect("Couldn't get caps from sample");
+        let structure = caps.get_structure(0)
+            .expect("Couldn't get structure from audio sample");
+        let rate = structure.get::<i32>("rate")
+            .expect("Couldn't get rate from audio sample");
+        // FIXME: channels is set in appsink
+        let channels = structure.get::<i32>("channels")
+            .expect("Couldn't get channels from audio sample")
+            as usize;
+        let channels_f = channels as f64;
+
+        let buffer = sample.get_buffer()
+            .expect("Couldn't get buffer from audio sample");
+
+        // assert_eq!(format, S16);
+
+        let sample_duration = 1_000_000_000 / (rate as u64);
 
         let map = buffer.map_readable().unwrap();
-        let data = map.as_slice();
+        let data = map.as_slice().as_slice_of::<i16>()
+            .expect("Couldn't get audio samples as i16");
+        let sample_nb = data.len() / channels;
+
+        let mut this = AudioBuffer {
+            sample_duration: sample_duration,
+            pts: buffer.get_pts(),
+            duration: buffer.get_duration(),
+            samples: Vec::with_capacity(sample_nb),
+        };
 
-        let mut data_reader = Cursor::new(data);
-        let channels_f = info.channels() as f64;
-        let mut keep_going = true;
-        while keep_going {
+        for index in 0..sample_nb {
             let mut mono_sample = 0f64;
-            for _ in 0..info.channels() {
-                let norm_sample = match info.format() {
-                    AudioFormat::F32le => {
-                        data_reader.read_f32::<LittleEndian>().map(|v| v as f64)
-                    },
-                    AudioFormat::F64le => {
-                        data_reader.read_f64::<LittleEndian>()
-                    },
-                    AudioFormat::S16le => {
-                        data_reader.read_i16::<LittleEndian>().map(|v|
-                            v as f64 / ::std::i16::MAX as f64
-                        )
-                    },
-                    AudioFormat::S32le => {
-                        data_reader.read_i32::<LittleEndian>().map(|v|
-                            v as f64 / ::std::i32::MAX as f64
-                        )
-                    },
-                    AudioFormat::U8 => {
-                        data_reader.read_u8().map(|v|
-                            (v as f64 - ::std::i8::MAX as f64) / ::std::i8::MAX as f64
-                        )
-                    },
-                    _ => panic!("never happens"), // FIXME: use proper assert
-                };
-
-                match norm_sample {
-                    Ok(norm_sample) => mono_sample += norm_sample,
-                    Err(_) => {
-                        keep_going = false;
-                        break;
-                    },
-                }
+            // FIXME: downmix in the pipeline (maybe done by appsink)
+            for channel in 0..channels {
+                mono_sample += data[index + channel] as f64;
             }
 
-            if keep_going {
-                this.samples.push(1f64 - (mono_sample / channels_f));
-            }
+            this.samples.push(1f64 - (mono_sample / (i16::MAX as f64) / channels_f));
         }
 
         this