Added opus streaming

chris-rudmin · Apr 13, 2015 · 4c78b1f · 4c78b1f
1 parent 2c1ab2a
commit 4c78b1f
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 145 deletions.
diff --git a/README.md b/README.md
@@ -18,8 +18,9 @@ Creates a recorder instance. Instantiating an instance will prompt the user for
 - **bufferLength** - (*optional*) The length of the buffer that the internal JavaScriptNode uses to capture the audio. Can be tweaked if experiencing performance issues. Defaults to 4096.
 - **monitorGain** - (*optional*) Sets the gain of the monitoring output. Gain is an a-weighted value between 0 and 1. Defaults to 0
 - **numberOfChannels** - (*optional*) The number of channels to record. 1 = mono, 2 = stereo. Defaults to 1. More than two channels has not been tested.
-- **recordOpus** - (*optional*) Specifies if recorder should record using the opus encoder. Defaults to true.
-- **sampleRate** - (*optional*) Specifies the sample rate to record at. Defaults to device sample rate. If resampling occurs, the audio is filtered with a 6th order butterworth filter and then resampled using a gaussing convolution. The Opus encoder will not work if the value is not 8000, 12000, 16000, 24000 or 48000.
+- **recordOpus** - (*optional*) Specifies if recorder should record using the opus encoder. Defaults to true. If set to { stream : true }, then dataAvailable event will fire when each page is ready. Additionaly if stream is true, you can specify a maximum number of buffers per page to reduce latency using { stream: true, maxBuffersPerPage: 10 }. At 44100 Hz, 10 buffers will be ~1 second of latency.
+- **sampleRate** - (*optional*) Specifies the sample rate to record at. Defaults to device sample rate. If different than native rate, the audio will be filtered and resampled. If recordOpus is true, this value will default to 48000.
+The Opus encoder will not work if the value is not 8000, 12000, 16000, 24000 or 48000.
 - **workerPath** - (*optional*) Path to recorder.js worker script. Defaults to 'recorderWorker.js'
 
 
@@ -44,7 +45,7 @@ Creates a recorder instance. Instantiating an instance will prompt the user for
 
     rec.requestData()
 
-**requestData** will request the recorded data if not recording. If successful, the event "dataAvailable" will be published with a blob containing the appropriate data as an ogg or wav file depending on config.
+**requestData** will request the recorded data if not recording. If successful, the event "dataAvailable" will be published with a blob containing the appropriate data as an ogg or wav file depending on config. requestData will not work if recordOpus stream is enabled, as the data is being streamed and not recorded.
 
     rec.resume()
 

diff --git a/oggopus.js b/oggopus.js
@@ -1,39 +1,34 @@
 importScripts( 'libopus.js', 'wavepcm.js' );
 
 var OggOpus = function( config ){
-
   this.numberOfChannels = config.numberOfChannels;
   this.inputSampleRate = config.inputSampleRate;
   this.outputSampleRate = config.outputSampleRate;
-  this.bitDepth = config.bitDepth;
-  this.encoderApplication = config.encoderApplication || 2049; // 2048 = Voice, 2049 = Full Band Audio, 2051 = Restricted Low Delay
+  this.onPageComplete = config.onPageComplete || this.onPageComplete;
+  this.maxBuffersPerPage = config.recordOpus.maxBuffersPerPage || 40; // Limit latency for streaming
+  this.encoderApplication = config.recordOpus.encoderApplication || 2049; // 2048 = Voice, 2049 = Full Band Audio, 2051 = Restricted Low Delay
   this.encoderFrameSize = config.encoderFrameSize || 20; // 20ms frame
-  this.granuleIncrement = 48 * this.encoderFrameSize
   this.wavepcm = new WavePCM( config );
-  this.packets = [];
+
+  this.pageIndex = 0;
+  this.granulePosition = 0;
+  this.segmentData = new Uint8Array( 65025 );
+  this.segmentDataIndex = 0;
+  this.segmentTable = new Uint8Array( 255 );
+  this.segmentTableIndex = 0;
+  this.pages = [];
+  this.fileLength = 0;
+  this.buffersInPage = 0;
 
   this.initChecksumTable();
   this.initCodec();
-};
-
-OggOpus.prototype.decode = function( packets ) {
-  var outputSampleLength;
-  var decodedAudio = new Int16Array( packets.length * this.encoderBufferLength );
-
-  for ( var i = 0; i < packets.length; i++ ) {
-    this.decoderBuffer.set( packets[i] );
-    outputSampleLength = _opus_decode( this.decoder, this.decoderBufferPointer, packets[i].byteLength, this.decoderOutputPointer, this.decoderOutputMaxLength, 0);
-    decodedAudio.set( this.decoderOutputBuffer, i*this.encoderBufferLength );
-  }
-
-  return new Uint8Array( decodedAudio.buffer );
+  this.generateIdPage();
+  this.generateCommentPage();
 };
 
 OggOpus.prototype.encode = function( samples ) {
-  var outputPackets = [];
   var sampleIndex = 0;
   var lengthToCopy;
-  var outputPacketLength;
 
   while ( sampleIndex < samples.length ) {
 
@@ -43,13 +38,22 @@ OggOpus.prototype.encode = function( samples ) {
     this.encoderBufferIndex += lengthToCopy;
 
     if ( this.encoderBufferIndex === this.encoderBufferLength ) {
-      outputPacketLength = _opus_encode_float( this.encoder, this.encoderBufferPointer, this.encoderSamplesPerChannelPerPacket, this.encoderOutputPointer, this.encoderOutputMaxLength );
-      outputPackets.push( new Uint8Array( this.encoderOutputBuffer.subarray(0, outputPacketLength) ) );
+      var packetLength = _opus_encode_float( this.encoder, this.encoderBufferPointer, this.encoderSamplesPerChannelPerPacket, this.encoderOutputPointer, this.encoderOutputMaxLength );
+      this.segmentPacket( packetLength );
       this.encoderBufferIndex = 0;
     }
   }
 
-  return outputPackets;
+  this.buffersInPage++;
+  if ( this.buffersInPage >= this.maxBuffersPerPage ) {
+    this.generatePage();
+  }
+};
+
+OggOpus.prototype.encodeFinalFrame = function() {
+  this.encode( new Float32Array( this.encoderBufferLength - this.encoderBufferIndex ) );
+  this.headerType += 4;
+  this.generatePage();
 };
 
 OggOpus.prototype.getChecksum = function( data ){
@@ -60,51 +64,22 @@ OggOpus.prototype.getChecksum = function( data ){
   return checksum >>> 0;
 };
 
-OggOpus.prototype.getCommentPage = function( pageIndex ){
-  var segmentDataBuffer = new ArrayBuffer( 24 );
-  var segmentDataView = new DataView( segmentDataBuffer );
-  var segmentData = new Uint8Array( segmentDataBuffer );
-  var segmentTable = new Uint8Array(1);
-
-  segmentTable[0] = segmentData.length;
+OggOpus.prototype.generateCommentPage = function(){
+  var segmentDataView = new DataView( this.segmentData.buffer );
   segmentDataView.setUint32( 0, 1332770163, false ) // Magic Signature 'Opus'
   segmentDataView.setUint32( 4, 1415669619, false ) // Magic Signature 'Tags'
   segmentDataView.setUint32( 8, 8, true ); // Vendor Length
   segmentDataView.setUint32( 12, 1382376303, false ); // Vendor name 'Reco'
   segmentDataView.setUint32( 16, 1919182194, false ); // Vendor name 'rder'
   segmentDataView.setUint32( 20, 0, true ); // User Comment List Length
-
-  return this.getPage( 0, 0, pageIndex, segmentTable, segmentData );
+  this.segmentTableIndex = 1;
+  this.segmentDataIndex = this.segmentTable[0] = 24;
+  this.headerType = 0;
+  this.generatePage();
 };
 
-OggOpus.prototype.getFile = function( pageData ){
-  var lastPage = pageData[ pageData.length-1 ];
-  var oggFile = new Uint8Array( lastPage.fileOffset + lastPage.segmentTable.length + lastPage.segmentData.length + 27 );
-  var oggPageIndex = 0;
-
-  oggFile.set( this.getIdPage( oggPageIndex++ ) );
-  oggFile.set( this.getCommentPage( oggPageIndex++ ), 47 );
-
-  for ( var i = 0; i < pageData.length; i++ ) {
-    oggFile.set( this.getPage(
-      pageData[i].headerType,
-      pageData[i].granulePosition,
-      oggPageIndex++,
-      pageData[i].segmentTable,
-      pageData[i].segmentData
-    ), pageData[i].fileOffset );
-  }
-
-  return oggFile;
-};
-
-OggOpus.prototype.getIdPage = function( pageIndex ){
-  var segmentDataBuffer = new ArrayBuffer( 19 );
-  var segmentDataView = new DataView( segmentDataBuffer );
-  var segmentData = new Uint8Array( segmentDataBuffer );
-  var segmentTable = new Uint8Array(1);
-
-  segmentTable[0] = segmentData.length;
+OggOpus.prototype.generateIdPage = function(){
+  var segmentDataView = new DataView( this.segmentData.buffer );
   segmentDataView.setUint32( 0, 1332770163, false ) // Magic Signature 'Opus'
   segmentDataView.setUint32( 4, 1214603620, false ) // Magic Signature 'Head'
   segmentDataView.setUint8( 8, 1, true ); // Version
@@ -113,19 +88,21 @@ OggOpus.prototype.getIdPage = function( pageIndex ){
   segmentDataView.setUint32( 12, this.inputSampleRate, true ); // original sample rate
   segmentDataView.setUint16( 16, 0, true ); // output gain
   segmentDataView.setUint8( 18, 0, true ); // channel map 0 = mono or stereo
-
-  return this.getPage( 2, 0, pageIndex, segmentTable, segmentData );
+  this.segmentTableIndex = 1;
+  this.segmentDataIndex = this.segmentTable[0] = 19;
+  this.headerType = 2;
+  this.generatePage();
 };
 
-OggOpus.prototype.getPage = function( headerType, granulePosition, pageIndex, segmentTable, segmentData ){
-  var numberOfSegments = segmentTable.length;
-  var pageBuffer = new ArrayBuffer(  27 + numberOfSegments + segmentData.length );
+OggOpus.prototype.generatePage = function(){
+  var granulePosition = ( this.lastPositiveGranulePosition === this.granulePosition) ? -1 : this.granulePosition;
+  var pageBuffer = new ArrayBuffer(  27 + this.segmentTableIndex + this.segmentDataIndex );
   var pageBufferView = new DataView( pageBuffer );
   var page = new Uint8Array( pageBuffer );
 
   pageBufferView.setUint32( 0, 1332176723, false); // Capture Pattern starts all page headers 'OggS'
   pageBufferView.setUint8( 4, 0, true ); // Version
-  pageBufferView.setUint8( 5, headerType, true ); // 1 = continuation, 2 = beginning of stream, 4 = end of stream
+  pageBufferView.setUint8( 5, this.headerType, true ); // 1 = continuation, 2 = beginning of stream, 4 = end of stream
 
   // Number of samples upto and including this page at 48000Hz, into 64 bits
   pageBufferView.setUint32( 6, granulePosition, true );
@@ -134,13 +111,19 @@ OggOpus.prototype.getPage = function( headerType, granulePosition, pageIndex, se
   }
 
   pageBufferView.setUint32( 14, 0, true ); // Bitstream serial number
-  pageBufferView.setUint32( 18, pageIndex, true ); // Page sequence number
-  pageBufferView.setUint8( 26, numberOfSegments, true ); // Number of segments in page.
-  page.set( segmentTable, 27 ); // Segment Table
-  page.set( segmentData, 27 + numberOfSegments ); // Segment Data
+  pageBufferView.setUint32( 18, this.pageIndex++, true ); // Page sequence number
+  pageBufferView.setUint8( 26, this.segmentTableIndex, true ); // Number of segments in page.
+  page.set( this.segmentTable.subarray(0, this.segmentTableIndex), 27 ); // Segment Table
+  page.set( this.segmentData.subarray(0, this.segmentDataIndex), 27 + this.segmentTableIndex ); // Segment Data
   pageBufferView.setUint32( 22, this.getChecksum( page ), true ); // Checksum
 
-  return page;
+  this.onPageComplete( page );
+  this.segmentTableIndex = 0;
+  this.segmentDataIndex = 0;
+  this.buffersInPage = 0;
+  if ( granulePosition > 0 ) {
+    this.lastPositiveGranulePosition = granulePosition;
+  }
 };
 
 OggOpus.prototype.initChecksumTable = function(){
@@ -164,75 +147,43 @@ OggOpus.prototype.initCodec = function() {
   this.encoderOutputMaxLength = 4000; 
   this.encoderOutputPointer = _malloc( this.encoderOutputMaxLength );
   this.encoderOutputBuffer = HEAPU8.subarray( this.encoderOutputPointer, this.encoderOutputPointer + this.encoderOutputMaxLength );
+};
 
-  this.decoder = _opus_decoder_create( this.outputSampleRate, this.numberOfChannels, allocate(4, 'i32', ALLOC_STACK) );
-  this.decoderBufferPointer = _malloc( this.encoderOutputMaxLength );
-  this.decoderBuffer = HEAPU8.subarray( this.decoderBufferPointer, this.decoderBufferPointer + this.encoderOutputMaxLength );
-  this.decoderOutputMaxLength = this.encoderBufferLength * 2; // 2 bytes per sample
-  this.decoderOutputPointer = _malloc( this.decoderOutputMaxLength );
-  this.decoderOutputBuffer = HEAP16.subarray( this.decoderOutputPointer >> 1, (this.decoderOutputPointer + this.decoderOutputMaxLength) >> 1 );
+OggOpus.prototype.onPageComplete = function( page ){
+  this.fileLength += page.length;
+  this.pages.push( page );
 };
 
 OggOpus.prototype.recordBuffers = function( buffers ) {
-  this.packets.push.apply( this.packets, this.encode( this.wavepcm.resampleAndInterleave( buffers ) ) );
+  this.encode( this.wavepcm.resampleAndInterleave( buffers ) );
 };
 
-OggOpus.prototype.requestData = function(){
-  return this.getFile( this.segmentPackets( this.packets ) );
+OggOpus.prototype.requestData = function() {
+  var data = new Uint8Array( this.fileLength );
+  var offset = 0;
+  for ( var i = 0; i < this.pages.length; i++ ) {
+    data.set( this.pages[i], offset );
+    offset += this.pages[i].length;
+  }
+  return data;
 };
 
-OggOpus.prototype.segmentPackets = function( packets ) {
-  var segmentTable = new Uint8Array( 255 );
-  var segmentTableIndex = 0;
-  var segmentData = new Uint8Array( 65025 );
-  var segmentDataIndex = 0;
-  var granulePosition = 0;
-  var headerType = 0;
-  var lastPositiveGranulePosition = 0;
-  var segmentedPackets = [];
-  var fileOffset = 99; // size of comment and id page
-  var pageComplete = function(){
-
-    segmentedPackets.push({
-      segmentTable: new Uint8Array( segmentTable.subarray(0, segmentTableIndex) ),
-      segmentData: new Uint8Array( segmentData.subarray(0, segmentDataIndex) ),
-      headerType: headerType,
-      granulePosition: (lastPositiveGranulePosition === granulePosition) ? -1 : granulePosition,
-      fileOffset: fileOffset
-    });
-
-    fileOffset += 27 + segmentTableIndex + segmentDataIndex;
-    segmentTableIndex = 0;
-    segmentDataIndex = 0;
-    if ( segmentedPackets[segmentedPackets.length-1].granulePosition !== -1 ) {
-      lastPositiveGranulePosition = granulePosition;
-    }
-  };
-
-  for ( var i = 0; i < packets.length; i++ ) {
-    var remainingPacketLength = packets[i].length;
-    var packetIndex = 0;
-
-    while ( remainingPacketLength >= 0 ) {
+OggOpus.prototype.segmentPacket = function( packetLength ) {
+  var packetIndex = 0;
 
-      if ( segmentTableIndex === 255 ) {
-        pageComplete();
-        headerType = ( remainingPacketLength >= 255 ) ? 1 : 0;
-      }
+  while ( packetLength >= 0 ) {
+    var segmentLength = Math.min( packetLength, 255 );
+    this.segmentTable[ this.segmentTableIndex++ ] = segmentLength;
+    this.segmentData.set( this.encoderOutputBuffer.subarray( packetIndex, packetIndex + segmentLength ), this.segmentDataIndex );
+    this.segmentDataIndex += segmentLength;
+    packetIndex += segmentLength;
+    packetLength -= 255;
 
-      var dataLength = Math.min( remainingPacketLength, 255 );
-      segmentData.set( packets[i].subarray( packetIndex, dataLength ), segmentDataIndex );
-      segmentTable[ segmentTableIndex++ ] = dataLength;
-      packetIndex += dataLength;
-      segmentDataIndex += dataLength;
-      remainingPacketLength -= 255;
+    if ( this.segmentTableIndex === 255 ) {
+      this.generatePage();
+      this.headerType = ( packetLength >= 0 ) ? 1 : 0;
     }
-
-    granulePosition += this.granuleIncrement;
   }
 
-  headerType += 4;
-  pageComplete();
-
-  return segmentedPackets;
-};
+  this.granulePosition += ( 48 * this.encoderFrameSize );
+};
diff --git a/recorder.js b/recorder.js
@@ -8,8 +8,8 @@ var Recorder = function( config ){
   }
 
   config = config || {};
-  config.recordOpus = config.recordOpus === false ? false : true;
-  config.bitDepth = (config.recordOpus ? 16 : config.bitDepth) || 16;
+  config.recordOpus = (config.recordOpus === false) ? false : config.recordOpus || true;
+  config.bitDepth = config.recordOpus ? 16 : config.bitDepth || 16;
   config.bufferLength = config.bufferLength || 4096;
   config.monitorGain = config.monitorGain || 0;
   config.numberOfChannels = config.numberOfChannels || 1;
@@ -164,7 +164,7 @@ Recorder.prototype.stop = function(){
     this.scriptProcessorNode.disconnect();
     this.state = "inactive";
     this.eventTarget.dispatchEvent( new Event( 'stop' ) );
-    this.requestData();
+    this.worker.postMessage({ command: "requestData" });
     this.worker.postMessage({ command: "stop" });
   }
 };
diff --git a/recorderWorker.js b/recorderWorker.js
@@ -1,28 +1,38 @@
 this.onmessage = function( e ){
+  var worker = this;
   switch( e.data.command ){
 
     case 'recordBuffers':
-      this.recorder.recordBuffers( e.data.buffers );
+      worker.recorder.recordBuffers( e.data.buffers );
       break;
 
     case 'requestData':
-      var data = this.recorder.requestData();
-      this.postMessage( data, [data.buffer] );
+      if ( worker.recorder.encodeFinalFrame ) {
+        worker.recorder.encodeFinalFrame();
+      }
+      if ( !worker.recordOpus.stream ) {
+        var data = worker.recorder.requestData();
+        worker.postMessage( data, [data.buffer] );
+      }
       break;
 
     case 'stop':
-      this.close();
+      worker.close();
       break;
 
     case 'start':
-      if ( e.data.recordOpus ) {
+      worker.recordOpus = e.data.recordOpus;
+      if ( worker.recordOpus ) {
         importScripts( 'oggopus.js' );
-        this.recorder = new OggOpus( e.data );
+        if ( worker.recordOpus.stream ) {
+          e.data.onPageComplete = function( page ){ worker.postMessage( page, [page.buffer] ); };
+        }
+        worker.recorder = new OggOpus( e.data );
       }
       else {
         importScripts( 'wavepcm.js' );
-        this.recorder = new WavePCM( e.data );
+        worker.recorder = new WavePCM( e.data );
       }
       break;
   }
-};
+};
diff --git a/wavepcm.js b/wavepcm.js
@@ -137,7 +137,7 @@ WavePCM.prototype.resampleAndInterleave = function( buffers ) {
       var channelData = buffers[ channel ];
 
       for ( var tap = -1; tap < 2; tap++ ) {
-        var sampleValue = channelData[ nearestPoint + tap ] || this.cachedSamples[channel][ 1 + tap ];
+        var sampleValue = channelData[ nearestPoint + tap ] || this.cachedSamples[channel][ 1 + tap ] || channelData[ nearestPoint ];
         outputData[ i * this.numberOfChannels + channel ] += sampleValue * this.magicKernel( resampleValue - nearestPoint - tap );
       }
     }