screenshare: add rap processing for screen streams with audio (complements #11622)

record-and-playback/core/lib/recordandplayback/edl/audio.rb

@@ -41,6 +41,25 @@ def self.dump(edl)
         end
       end
 
+      def self.mixer(inputs, output_basename)
+        BigBlueButton.logger.debug "Mixing audio files"
+
+        ffmpeg_cmd = [*FFMPEG]
+        inputs.each do |input|
+          ffmpeg_cmd += ['-i', input]
+        end
+        ffmpeg_cmd += ['-filter_complex', "amix"]
+
+        output = "#{output_basename}.#{WF_EXT}"
+        ffmpeg_cmd += [*FFMPEG_WF_ARGS, output]
+
+        BigBlueButton.logger.info "Running audio mixer..."
+        exitstatus = BigBlueButton.exec_ret(*ffmpeg_cmd)
+        raise "ffmpeg failed, exit code #{exitstatus}" if exitstatus != 0
+
+        output
+      end
+
       def self.render(edl, output_basename)
         audioinfo = {}
 

record-and-playback/core/lib/recordandplayback/generators/audio.rb

@@ -73,5 +73,70 @@ def self.create_audio_edl(events, archive_dir)
       return audio_edl
     end
 
+    def self.create_deskshare_audio_edl(events, deskshare_dir)
+      audio_edl = []
+
+      initial_timestamp = BigBlueButton::Events.first_event_timestamp(events)
+      final_timestamp = BigBlueButton::Events.last_event_timestamp(events)
+      filename = ""
+
+      # Initially start with silence
+      audio_edl << {
+        :timestamp => 0,
+        :audio => nil
+      }
+
+      events.xpath('/recording/event[@module="Deskshare" or (@module="bbb-webrtc-sfu" and (@eventname="StartWebRTCDesktopShareEvent" or @eventname="StopWebRTCDesktopShareEvent"))]').each do |event|
+        filename = event.at_xpath('filename').text
+        # Determine the audio filename
+        case event['eventname']
+        when 'DeskshareStartedEvent', 'DeskshareStoppedEvent'
+          filename = event.at_xpath('file').text
+          filename = "#{deskshare_dir}/#{File.basename(filename)}"
+        when 'StartWebRTCDesktopShareEvent', 'StopWebRTCDesktopShareEvent'
+          uri = event.at_xpath('filename').text
+          filename = "#{deskshare_dir}/#{File.basename(uri)}"
+        end
+        raise "Couldn't determine audio filename" if filename.nil?
+        # check if deskshare has audio
+        fileHasAudio = !BigBlueButton::EDL::Audio.audio_info(filename)[:audio].nil?
+        if (fileHasAudio)
+          timestamp = event['timestamp'].to_i - initial_timestamp
+          # Add the audio to the EDL
+          case event['eventname']
+          when 'DeskshareStartedEvent', 'StartWebRTCDesktopShareEvent'
+            audio_edl << {
+              :timestamp => timestamp,
+              :audio => { :filename => filename, :timestamp => 0 }
+            }
+          when 'DeskshareStoppedEvent', 'StopWebRTCDesktopShareEvent'
+            if audio_edl.last[:audio] && audio_edl.last[:audio][:filename] == filename
+              # Fill in the original/expected audo duration when available
+              duration = event.at_xpath('duration')
+              if !duration.nil?
+                duration = duration.text.to_i
+                audio_edl.last[:original_duration] = duration * 1000
+              else
+                audio_edl.last[:original_duration] = timestamp - audio_edl.last[:timestamp]
+              end
+              audio_edl << {
+                :timestamp => timestamp,
+                :audio => nil
+              }
+            end
+          end
+        else
+          BigBlueButton.logger.debug " Screenshare without audio, ignoring..."
+        end
+      end
+
+      audio_edl << {
+        :timestamp => final_timestamp - initial_timestamp,
+        :audio => nil
+      }
+
+      return audio_edl
+    end
+
   end
 end

record-and-playback/core/lib/recordandplayback/generators/audio_processor.rb

@@ -36,7 +36,6 @@ class AudioProcessor
     def self.process(archive_dir, file_basename)
       BigBlueButton.logger.info("AudioProcessor.process: Processing audio...")
 
-      audio_dir = "#{archive_dir}/audio"
       events_xml = "#{archive_dir}/events.xml"
       events = Nokogiri::XML(File.open(events_xml))
 
@@ -52,12 +51,36 @@ def self.process(archive_dir, file_basename)
       BigBlueButton::EDL::Audio.dump(audio_edl)
 
       target_dir = File.dirname(file_basename)
-      audio_dir = "#{archive_dir}/audio"
-      events_xml = "#{archive_dir}/events.xml"
 
+      # getting users audio...
       @audio_file = BigBlueButton::EDL::Audio.render(
         audio_edl, File.join(target_dir, 'recording'))
 
+      # and mixing it with deskshare audio	
+      deskshare_dir = "#{archive_dir}/deskshare"
+      if BigBlueButton::Events.screenshare_has_audio?(events_xml, deskshare_dir)
+        BigBlueButton.logger.info("AudioProcessor.process: processing Deskshare audio...")	
+
+        mixed_dir = "#{archive_dir}/mixed"
+
+        deskshare_audio_edl = BigBlueButton::AudioEvents.create_deskshare_audio_edl(events, deskshare_dir)
+        BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)	
+
+        BigBlueButton.logger.info "Applying recording start/stop events to Deskshare audio"
+        deskshare_audio_edl = BigBlueButton::Events.edl_match_recording_marks_audio(
+          deskshare_audio_edl, events, start_time, end_time)
+        BigBlueButton.logger.debug "Trimmed Deskshare Audio EDL:"
+        BigBlueButton::EDL::Audio.dump(deskshare_audio_edl)
+
+        audio_inputs = []	
+        audio_inputs << @audio_file	
+        audio_inputs << BigBlueButton::EDL::Audio.render(deskshare_audio_edl, deskshare_dir)	
+
+        @audio_file = BigBlueButton::EDL::Audio.mixer(audio_inputs, mixed_dir)	
+      else
+        BigBlueButton.logger.info("AudioProcessor.process: no Deskshare audio to process.")	
+      end
+
       ogg_format = {
         :extension => 'ogg',
         :parameters => [ [ '-c:a', 'copy', '-f', 'ogg' ] ]

record-and-playback/core/lib/recordandplayback/generators/events.rb

@@ -692,5 +692,19 @@ def self.bbb_version_compare(events, major, minor=nil, micro=nil)
       end
     end
 
+    # Check if any screenshare files has audio
+    def self.screenshare_has_audio?(events_xml, deskshare_dir)
+      events = Nokogiri::XML(File.open(events_xml))
+      events.xpath('/recording/event[@eventname="StartWebRTCDesktopShareEvent"]').each do |event|
+        filename = event.at_xpath('filename').text
+        filename = "#{deskshare_dir}/#{File.basename(filename)}"
+        fileHasAudio = !BigBlueButton::EDL::Audio.audio_info(filename)[:audio].nil?
+        if fileHasAudio
+          return true
+        end
+      end
+      return false
+    end
+
   end
 end