local API: Support multiple audio tracks

src/renderer/components/ft-video-player/ft-video-player.js

@@ -82,13 +82,18 @@ export default defineComponent({
     chapters: {
       type: Array,
       default: () => { return [] }
+    },
+    audioTracks: {
+      type: Array,
+      default: () => ([])
     }
   },
   data: function () {
     return {
       powerSaveBlocker: null,
       volume: 1,
       muted: false,
+      /** @type {(import('video.js').VideoJsPlayer|null)} */
       player: null,
       useDash: false,
       useHls: false,
@@ -129,6 +134,7 @@ export default defineComponent({
             'screenshotButton',
             'playbackRateMenuButton',
             'loopButton',
+            'audioTrackButton',
             'chaptersButton',
             'descriptionsButton',
             'subsCapsButton',
@@ -309,6 +315,24 @@ export default defineComponent({
       this.toggleScreenshotButton()
     }
   },
+  created: function () {
+    this.dataSetup.playbackRates = this.playbackRates
+
+    if (this.format === 'audio') {
+      // hide the PIP button for the audio formats
+      const controlBarItems = this.dataSetup.controlBar.children
+      const index = controlBarItems.indexOf('pictureInPictureToggle')
+      controlBarItems.splice(index, 1)
+    }
+
+    if (this.format === 'legacy' || this.audioTracks.length === 0) {
+      // hide the audio track selector for the legacy formats
+      // and Invidious(it doesn't give us the information for multiple audio tracks yet)
+      const controlBarItems = this.dataSetup.controlBar.children
+      const index = controlBarItems.indexOf('audioTrackButton')
+      controlBarItems.splice(index, 1)
+    }
+  },
   mounted: function () {
     const volume = sessionStorage.getItem('volume')
     const muted = sessionStorage.getItem('muted')
@@ -323,8 +347,6 @@ export default defineComponent({
       this.muted = (muted === 'true')
     }
 
-    this.dataSetup.playbackRates = this.playbackRates
-
     if (this.format === 'dash') {
       this.determineDefaultQualityDash()
     }
@@ -370,13 +392,6 @@ export default defineComponent({
           await this.determineDefaultQualityLegacy()
         }
 
-        if (this.format === 'audio') {
-          // hide the PIP button for the audio formats
-          const controlBarItems = this.dataSetup.controlBar.children
-          const index = controlBarItems.indexOf('pictureInPictureToggle')
-          controlBarItems.splice(index, 1)
-        }
-
         // regardless of what DASH qualities you enable or disable in the qualityLevels plugin
         // the first segments videojs-http-streaming requests are chosen based on the available bandwidth, which is set to 0.5MB/s by default
         // overriding that to be the same as the quality we requested, makes videojs-http-streamming pick the correct quality
@@ -427,6 +442,36 @@ export default defineComponent({
           })
         }
 
+        // for the DASH formats, videojs-http-streaming takes care of the audio track management for us,
+        // thanks to the values in the DASH manifest
+        // so we only need a custom implementation for the audio only formats
+        if (this.format === 'audio' && this.audioTracks.length > 0) {
+          /** @type {import('../../views/Watch/Watch.js').AudioTrack[]} */
+          const audioTracks = this.audioTracks
+
+          const audioTrackList = this.player.audioTracks()
+          audioTracks.forEach(({ id, kind, label, language, isDefault: enabled }) => {
+            audioTrackList.addTrack(new videojs.AudioTrack({
+              id, kind, label, language, enabled,
+            }))
+          })
+
+          audioTrackList.on('change', () => {
+            let trackId
+            // doesn't support foreach so we need to use an indexed for loop here
+            for (let i = 0; i < audioTrackList.length; i++) {
+              const track = audioTrackList[i]
+
+              if (track.enabled) {
+                trackId = track.id
+                break
+              }
+            }
+
+            this.changeAudioTrack(trackId)
+          })
+        }
+
         this.player.volume(this.volume)
         this.player.muted(this.muted)
         this.player.playbackRate(this.defaultPlayback)
@@ -835,6 +880,45 @@ export default defineComponent({
       }
     },
 
+    /**
+     * @param {string} trackId
+     */
+    changeAudioTrack: function (trackId) {
+      // changing the player sources resets it, so we need to store the values that get reset,
+      // before we change the sources and restore them afterwards
+      const isPaused = this.player.paused()
+      const currentTime = this.player.currentTime()
+      const playbackRate = this.player.playbackRate()
+      const selectedQualityIndex = this.player.currentSources().findIndex(quality => quality.selected)
+
+      const newSourceList = this.audioTracks.find(audioTrack => audioTrack.id === trackId).sourceList
+
+      // video.js doesn't pick up changes to the sources in the HTML after it's initial load
+      // updating the sources of an existing player requires calling player.src() instead
+      // which accepts a different object that what use for the html sources
+
+      const newSources = newSourceList.map((source, index) => {
+        return {
+          src: source.url,
+          type: source.type,
+          label: source.qualityLabel,
+          selected: index === selectedQualityIndex
+        }
+      })
+
+      this.player.one('canplay', () => {
+        this.player.currentTime(currentTime)
+        this.player.playbackRate(playbackRate)
+
+        // need to call play to restore the player state, even if we want to pause afterwards
+        this.player.play().then(() => {
+          if (isPaused) { this.player.pause() }
+        })
+      })
+
+      this.player.src(newSources)
+    },
+
     determineFormatType: function () {
       if (this.format === 'dash') {
         this.enableDashFormat()

src/renderer/views/Watch/Watch.js

@@ -31,6 +31,22 @@ import {
 } from '../../helpers/api/local'
 import { filterInvidiousFormats, invidiousGetVideoInformation, youtubeImageUrlToInvidious } from '../../helpers/api/invidious'
 
+/**
+ * @typedef {object} AudioSource
+ * @property {string} url
+ * @property {string} type
+ * @property {string} label
+ * @property {string} qualityLabel
+ *
+ * @typedef {object} AudioTrack
+ * @property {string} id
+ * @property {('main'|'translation'|'descriptions'|'alternative')} kind - https://videojs.com/guides/audio-tracks/#kind
+ * @property {string} label
+ * @property {string} language
+ * @property {boolean} isDefault
+ * @property {AudioSource[]} sourceList
+ */
+
 export default defineComponent({
   name: 'Watch',
   components: {
@@ -88,6 +104,10 @@ export default defineComponent({
       activeSourceList: [],
       videoSourceList: [],
       audioSourceList: [],
+      /**
+       * @type {AudioTrack[]}
+       */
+      audioTracks: [],
       adaptiveFormats: [],
       captionHybridList: [], // [] -> Promise[] -> string[] (URIs)
       recommendedVideos: [],
@@ -196,6 +216,7 @@ export default defineComponent({
       this.captionHybridList = []
       this.downloadLinks = []
       this.videoCurrentChapterIndex = 0
+      this.audioTracks = []
 
       this.checkIfPlaylist()
       this.checkIfTimestamp()
@@ -557,32 +578,68 @@ export default defineComponent({
           }
 
           if (result.streaming_data?.adaptive_formats.length > 0) {
-            this.audioSourceList = result.streaming_data.adaptive_formats.filter((format) => {
+            const audioFormats = result.streaming_data.adaptive_formats.filter((format) => {
               return format.has_audio
-            }).sort((a, b) => {
-              return a.bitrate - b.bitrate
-            }).map((format, index) => {
-              const label = (x) => {
-                switch (x) {
-                  case 0:
-                    return this.$t('Video.Audio.Low')
-                  case 1:
-                    return this.$t('Video.Audio.Medium')
-                  case 2:
-                    return this.$t('Video.Audio.High')
-                  case 3:
-                    return this.$t('Video.Audio.Best')
-                  default:
-                    return format.bitrate
+            })
+
+            const hasMultipleAudioTracks = audioFormats.some(format => format.audio_track)
+
+            if (hasMultipleAudioTracks) {
+              /** @type {string[]} */
+              const ids = []
+
+              /** @type {AudioTrack[]} */
+              const audioTracks = []
+
+              /** @type {import('youtubei.js').Misc.Format[][]} */
+              const sourceLists = []
+
+              audioFormats.forEach(format => {
+                const index = ids.indexOf(format.audio_track.id)
+                if (index === -1) {
+                  ids.push(format.audio_track.id)
+
+                  let kind
+
+                  if (format.audio_track.audio_is_default) {
+                    kind = 'main'
+                  } else if (format.is_dubbed) {
+                    kind = 'translation'
+                  } else if (format.is_descriptive) {
+                    kind = 'descriptions'
+                  } else {
+                    kind = 'alternative'
+                  }
+
+                  audioTracks.push({
+                    id: format.audio_track.id,
+                    kind,
+                    label: format.audio_track.display_name,
+                    language: format.language,
+                    isDefault: format.audio_track.audio_is_default,
+                    sourceList: []
+                  })
+
+                  sourceLists.push([
+                    format
+                  ])
+                } else {
+                  sourceLists[index].push(format)
                 }
+              })
+
+              for (let i = 0; i < audioTracks.length; i++) {
+                audioTracks[i].sourceList = this.createLocalAudioSourceList(sourceLists[i])
               }
-              return {
-                url: format.url,
-                type: format.mime_type,
-                label: 'Audio',
-                qualityLabel: label(index)
-              }
-            }).reverse()
+
+              this.audioTracks = audioTracks
+
+              this.audioSourceList = this.audioTracks.find(track => track.isDefault).sourceList
+            } else {
+              this.audioTracks = []
+
+              this.audioSourceList = this.createLocalAudioSourceList(audioFormats)
+            }
 
             // we need to alter the result object so the toDash function uses the filtered formats too
             result.streaming_data.adaptive_formats = filterLocalFormats(result.streaming_data.adaptive_formats, this.allowDashAv1Formats)
@@ -626,6 +683,8 @@ export default defineComponent({
         console.error(err)
         if (this.backendPreference === 'local' && this.backendFallback && !err.toString().includes('private')) {
           showToast(this.$t('Falling back to Invidious API'))
+          // Invidious doesn't support multiple audio tracks, so we need to clear this to prevent the player getting confused
+          this.audioTracks = []
           this.getVideoInformationInvidious()
         } else {
           this.isLoading = false
@@ -881,6 +940,42 @@ export default defineComponent({
       }
     },
 
+    /**
+     * @param {import('youtubei.js').Misc.Format[]} audioFormats
+     * @returns {AudioSource[]}
+     */
+    createLocalAudioSourceList: function (audioFormats) {
+      return audioFormats.sort((a, b) => {
+        return a.bitrate - b.bitrate
+      }).map((format, index) => {
+        let label
+
+        switch (index) {
+          case 0:
+            label = this.$t('Video.Audio.Low')
+            break
+          case 1:
+            label = this.$t('Video.Audio.Medium')
+            break
+          case 2:
+            label = this.$t('Video.Audio.High')
+            break
+          case 3:
+            label = this.$t('Video.Audio.Best')
+            break
+          default:
+            label = format.bitrate.toString()
+        }
+
+        return {
+          url: format.url,
+          type: format.mime_type,
+          label: 'Audio',
+          qualityLabel: label
+        }
+      }).reverse()
+    },
+
     addToHistory: function (watchProgress) {
       const videoData = {
         videoId: this.videoId,

src/renderer/views/Watch/Watch.vue

@@ -21,6 +21,7 @@
           ref="videoPlayer"
           :dash-src="dashSrc"
           :source-list="activeSourceList"
+          :audio-tracks="audioTracks"
           :adaptive-formats="adaptiveFormats"
           :caption-hybrid-list="captionHybridList"
           :storyboard-src="videoStoryboardSrc"