Merge pull request #11683 from kepstin/audio-rework

Recording: Audio processing changes to reduce audio desync

record-and-playback/core/lib/recordandplayback/edl/audio.rb

@@ -21,7 +21,7 @@ module BigBlueButton
   module EDL
     module Audio
       FFMPEG_AEVALSRC = "aevalsrc=s=48000:c=stereo:exprs=0|0"
-      FFMPEG_AFORMAT = "aformat=sample_fmts=s16:sample_rates=48000:channel_layouts=stereo"
+      FFMPEG_AFORMAT = "aresample=async=1000,aformat=sample_fmts=s16:sample_rates=48000:channel_layouts=stereo"
       FFMPEG_WF_CODEC = 'libvorbis'
       FFMPEG_WF_ARGS = ['-c:a', FFMPEG_WF_CODEC, '-q:a', '2', '-f', 'ogg']
       WF_EXT = 'ogg'
@@ -42,7 +42,6 @@ def self.dump(edl)
       end
 
       def self.render(edl, output_basename)
-        sections = []
         audioinfo = {}
 
         corrupt_audios = Set.new
@@ -61,13 +60,14 @@ def self.render(edl, output_basename)
         audioinfo.keys.each do |audiofile|
           BigBlueButton.logger.debug "  #{audiofile}"
           info = audio_info(audiofile)
-          BigBlueButton.logger.debug "    sample rate: #{info[:sample_rate]}, duration: #{info[:duration]}"
-
           if !info[:audio] || !info[:duration]
             BigBlueButton.logger.warn "    This audio file is corrupt! It will be removed from the output."
             corrupt_audios << audiofile
           end
 
+          BigBlueButton.logger.debug "    format: #{info[:format][:format_name]}, codec: #{info[:audio][:codec_name]}"
+          BigBlueButton.logger.debug "    sample rate: #{info[:sample_rate]}, duration: #{info[:duration]}"
+
           audioinfo[audiofile] = info
         end
 
@@ -82,70 +82,60 @@ def self.render(edl, output_basename)
           dump(edl)
         end
 
-        input_index = 0
-        output_index = 0
         ffmpeg_inputs = []
-        ffmpeg_filters = []
+        ffmpeg_filter = ''
         BigBlueButton.logger.info "Generating ffmpeg command"
         for i in 0...(edl.length - 1)
           entry = edl[i]
           audio = entry[:audio]
           duration = entry[:next_timestamp] - entry[:timestamp]
 
-          # Check for and handle audio files with mismatched lengths (generated
-          # by buggy versions of freeswitch in old BigBlueButton
-          if audio and entry[:original_duration] and
-               (audioinfo[audio[:filename]][:duration].to_f / entry[:original_duration]) < 0.997 and
-               ((entry[:original_duration] - audioinfo[audio[:filename]][:duration]).to_f /
-                      entry[:original_duration]).abs < 0.05
-            speed = audioinfo[audio[:filename]][:duration].to_f / entry[:original_duration]
-            BigBlueButton.logger.info "  Using input #{audio[:filename]}"
+          ffmpeg_filter << "[a_edl#{i}_prev];\n" if i > 0
 
-            BigBlueButton.logger.warn "  Audio file length mismatch, adjusting speed to #{speed}"
-
-            # Have to calculate the start point after the atempo filter in this case,
-            # since it can affect the audio start time.
-            # Also reset the pts to start at 0, so the duration trim works correctly.
-            filter = "[#{input_index}] "
-            filter << "atempo=#{speed},atrim=start=#{ms_to_s(audio[:timestamp])},"
-            filter << "asetpts=PTS-STARTPTS,"
-            filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
-            ffmpeg_filters << filter
-
-            ffmpeg_inputs << {
-              :filename => audio[:filename],
-              :seek => 0
-            }
-
-            input_index += 1
-            output_index += 1
-
-            # Normal audio input handling. Skip this input and generate silence
-            # if the seekpoint is past the end of the audio, which can happen
-            # if events are slightly misaligned and you get unlucky with a
-            # start/stop or chapter break.
-          elsif audio and audio[:timestamp] < audioinfo[audio[:filename]][:duration]
+          if audio
             BigBlueButton.logger.info "  Using input #{audio[:filename]}"
 
-            filter = "[#{input_index}] "
-            filter << "#{FFMPEG_AFORMAT},apad,atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
-            ffmpeg_filters << filter
+            speed = 1
+            seek = audio[:timestamp]
 
-            ffmpeg_inputs << {
-              :filename => audio[:filename],
-              :seek => audio[:timestamp]
-            }
+            # Check for and handle audio files with mismatched lengths (generated
+            # by buggy versions of freeswitch in old BigBlueButton
+            if entry[:original_duration] && (audioinfo[audio[:filename]][:duration].to_f / entry[:original_duration]) < 0.997 &&
+               ((entry[:original_duration] - audioinfo[audio[:filename]][:duration]).to_f /
+                      entry[:original_duration]).abs < 0.05
+              BigBlueButton.logger.warn "  Audio file length mismatch, adjusting speed to #{speed}"
+              speed = audioinfo[audio[:filename]][:duration].to_f / entry[:original_duration]
+              seek = 0
+            end
+
+            # Skip this input and generate silence if the seekpoint is past the end of the audio, which can happen
+            # if events are slightly misaligned and you get unlucky with a start/stop or chapter break.
+            if audio[:timestamp] < (audioinfo[audio[:filename]][:duration] * speed)
+              input_index = ffmpeg_inputs.length
+              ffmpeg_inputs << {
+                filename: audio[:filename],
+                seek: seek
+              }
+              ffmpeg_filter << "[#{input_index}]#{FFMPEG_AFORMAT},apad"
+            else
+              ffmpeg_filter << "#{FFMPEG_AEVALSRC},#{FFMPEG_AFORMAT}"
+            end
 
-            input_index += 1
-            output_index += 1
+            ffmpeg_filter << ",atempo=#{speed},atrim=start=#{ms_to_s(audio[:timestamp])}" if speed != 1
 
+            ffmpeg_filter << ",asetpts=PTS-STARTPTS"
           else
             BigBlueButton.logger.info "  Generating silence"
 
-            ffmpeg_filters << "#{FFMPEG_AEVALSRC},#{FFMPEG_AFORMAT},atrim=end=#{ms_to_s(duration)} [out#{output_index}]"
+            ffmpeg_filter << "#{FFMPEG_AEVALSRC},#{FFMPEG_AFORMAT}"
+          end
 
-            output_index += 1
+          if i > 0
+            ffmpeg_filter << "[a_edl#{i}];\n"
+            ffmpeg_filter << "[a_edl#{i}_prev][a_edl#{i}]concat=n=2:a=1:v=0"
           end
+
+          ffmpeg_filter << ",atrim=end=#{ms_to_s(entry[:next_timestamp])}"
         end
 
         ffmpeg_cmd = [*FFMPEG]
@@ -155,21 +145,21 @@ def self.render(edl, output_basename)
           if audioinfo[input[:filename]][:format][:format_name] == 'wav'
             ffmpeg_cmd += ['-ignore_length', '1']
           end
+          # Prefer using the libopus decoder for opus files, it handles discontinuities better
+          if audioinfo[input[:filename]][:audio][:codec_name] == 'opus'
+            ffmpeg_cmd << '-c:a' << 'libopus'
+          end
           ffmpeg_cmd += ['-i', input[:filename]]
         end
-        ffmpeg_filter = ffmpeg_filters.join(' ; ')
-
-        if output_index > 1
-          # Add the final concat filter
-          ffmpeg_filter << " ; "
-          (0...output_index).each { |i| ffmpeg_filter << "[out#{i}]" }
-          ffmpeg_filter << " concat=n=#{output_index}:a=1:v=0"
-        else
-          # Only one input, no need for concat filter
-          ffmpeg_filter << " ; [out0] anull"
+
+        BigBlueButton.logger.debug('  ffmpeg filter_complex_script:')
+        BigBlueButton.logger.debug(ffmpeg_filter)
+        filter_complex_script = "#{output_basename}.filter"
+        File.open(filter_complex_script, 'w') do |io|
+          io.write(ffmpeg_filter)
         end
 
-        ffmpeg_cmd += ['-filter_complex', ffmpeg_filter]
+        ffmpeg_cmd << '-filter_complex_script' << filter_complex_script
 
         output = "#{output_basename}.#{WF_EXT}"
         ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]