Supports iOS native audio playback as workaround for iOS 17.2.1 HTML …

…5 audio bug

Chavah.NetCore/wwwroot/js/App.ts

@@ -117,7 +117,6 @@
         "appNav",
         "adminScripts",
         "navigatorMediaSession",
-        "iOSMediaSession",
         "$rootScope",
         "$location",
         "$window",
@@ -128,7 +127,6 @@
             appNav: AppNavService,
             adminScripts: AdminScriptsService,
             navigatorMediaSession: NavigatorMediaSessionService,
-            iOSMediaSession: IOSMediaSessionService,
             $rootScope: ng.IRootScopeService,
             $location: ng.ILocationService,
             $window: ng.IWindowService,
@@ -138,8 +136,7 @@
             // See http://stackoverflow.com/a/41825004/536
             $window["Promise"] = $q;
 
-            // Integrate with the host platform's audio services, e.g. lockscreen media buttons, "currently playing" media info panels, etc.
-            iOSMediaSession.install(); // iOS
+            // Integrate with the host platform's audio services, e.g. lock screen media buttons, "currently playing" media info panels, etc.
             navigatorMediaSession.install(); // Android, emerging web standard
 
             // Attach the view-busted template paths to the root scope so that we can bind to the names in our views.

Chavah.NetCore/wwwroot/js/Controllers/FooterController.ts

@@ -127,21 +127,15 @@
         }
 
         get volume() {
-            if (this.audio) {
-                return this.audio.volume;
-            }
-
-            return 1;
+            return this.audioPlayer.volume;
         }
 
         set volume(val: number) {
             if (isNaN(val)) {
                 val = 1.0;
             }
-            if (this.audio) {
-                this.audio.volume = val;
-            }
 
+            this.audioPlayer.volume = val;
             this.volumeVal.onNext(val);
         }
 

Chavah.NetCore/wwwroot/js/Controllers/HeaderController.ts

@@ -159,6 +159,13 @@
         dismissDonationBanner() {
             window.localStorage.setItem(HeaderController.donationBannerLocalStorageKey, "true");
         }
+
+        skipToNearEnd(): void {
+            const duration = this.audioPlayer.duration.getValue();
+            if (!isNaN(duration) && duration > 0) {
+                this.audioPlayer.skipToNearEndZanz();
+            }
+        }
     }
 
     App.controller("HeaderController", HeaderController);

Chavah.NetCore/wwwroot/js/Services/AudioPlayerService.ts

@@ -4,7 +4,8 @@ namespace BitShuva.Chavah {
 
         static $inject = [
             "songApi",
-            "homeViewModel"
+            "homeViewModel",
+            "iosAudioPlayer"
         ];
 
         readonly status = new Rx.BehaviorSubject(AudioStatus.Paused);
@@ -16,40 +17,37 @@ namespace BitShuva.Chavah {
         readonly playedTimePercentage = new Rx.BehaviorSubject<number>(0);
         readonly duration = new Rx.BehaviorSubject<number>(0);
         playedSongs: Song[] = [];
-        private audio: HTMLAudioElement;
+        private audio: PlatformAudio;
         readonly error = new Rx.Subject<IAudioErrorInfo>();
 
         private lastPlayedTime = 0;
 
         constructor(
             private readonly songApi: SongApiService,
-            private readonly homeViewModel: Server.HomeViewModel) {
+            private readonly homeViewModel: Server.HomeViewModel,
+            private readonly iosAudioPlayer: IOSAudioPlayer) {
 
             // Listen for when the song changes and update the document title.
             this.song
                 .subscribe(song => this.updateDocumentTitle(song));
         }
 
         initialize(audio: HTMLAudioElement) {
-            let supportsMp3Audio = Modernizr.audio.mp3;
-            if (supportsMp3Audio) {
-                this.audio = audio;
-
-                // this.audio.addEventListener("abort", (args) => this.aborted(args));
-                this.audio.addEventListener("ended", () => this.ended());
-                this.audio.addEventListener("error", args => this.erred(args));
-                this.audio.addEventListener("pause", () => this.status.onNext(AudioStatus.Paused));
-                this.audio.addEventListener("play", () => this.status.onNext(AudioStatus.Playing));
-                this.audio.addEventListener("playing", () => this.status.onNext(AudioStatus.Playing));
-                this.audio.addEventListener("waiting", () => this.status.onNext(AudioStatus.Buffering));
-                this.audio.addEventListener("stalled", args => this.stalled(args));
-                this.audio.addEventListener("timeupdate", args => this.playbackPositionChanged(args));
-            } else {
-                // UPGRADE TODO
-                // require(["viewmodels/upgradeBrowserDialog"],(UpgradeBrowserDialog) => {
-                //    App.showDialog(new UpgradeBrowserDialog());
-                // });
+            this.audio = audio;
+
+            // On the Chavah iOS app, we won't actually use HTML5 audio. See IOSAudioPlayer for details why.
+            if (this.iosAudioPlayer.isIOSWebApp) {
+                this.audio = new IOSAudioPlayer();
             }
+
+            this.audio.addEventListener("ended", () => this.ended());
+            this.audio.addEventListener("error", args => this.erred(args));
+            this.audio.addEventListener("pause", () => this.status.onNext(AudioStatus.Paused));
+            this.audio.addEventListener("play", () => this.status.onNext(AudioStatus.Playing));
+            this.audio.addEventListener("playing", () => this.status.onNext(AudioStatus.Playing));
+            this.audio.addEventListener("waiting", () => this.status.onNext(AudioStatus.Buffering));
+            this.audio.addEventListener("stalled", args => this.stalled(args));
+            this.audio.addEventListener("timeupdate", args => this.playbackPositionChanged(args));
         }
 
         playNewSong(song: Song) {
@@ -198,25 +196,34 @@ namespace BitShuva.Chavah {
             }
         }
 
+        get volume(): number {
+            if (this.audio) {
+                return this.audio.volume;
+            }
+
+            return 1;
+        }
+
         /**
          * Sets the volume level.
          * @param level Should be between 0 and 1, where 1 is full volume, and 0 is muted.
          */
-        setVolume(level: number) {
+        set volume(level: number) {
             if (this.audio) {
-                this.audio.volume = 0;
+                this.audio.volume = level;
             }
         }
 
-        skipToEnd() {
+        skipToNearEndZanz() {
             if (this.audio && this.audio.duration) {
-                this.audio.currentTime = this.audio.duration - 1;
+                this.audio.currentTime = this.audio.duration - 10;
             }
         }
 
-        private aborted(args: any) {
-            this.status.onNext(AudioStatus.Aborted);
-            console.log("Audio aborted", this.audio.currentSrc, args);
+        skipToEnd() {
+            if (this.audio && this.audio.duration) {
+                this.audio.currentTime = this.audio.duration - 1;
+            }
         }
 
         private erred(args: ErrorEvent) {
@@ -244,7 +251,7 @@ namespace BitShuva.Chavah {
 
         private stalled(args: Event) {
             this.status.onNext(AudioStatus.Stalled);            
-            console.warn("Audio stalled, unable to stream in audio data.", this.audio.currentSrc, args);
+            console.warn("Audio stalled, unable to stream in audio data.", this.audio.src, args);
         }
 
         private playbackPositionChanged(args: any) {

Chavah.NetCore/wwwroot/js/Services/IOSAudioPlayer.ts

@@ -0,0 +1,120 @@
+namespace BitShuva.Chavah {
+    /**
+     * Provides an interface to play audio natively on iOS. Used by our webview-based iOS app.
+     * Why? Because as of 2024 (iOS 17.2.1), HTML audio cannot play new songs if the app is in the background or if the screen is off. See https://twitter.com/JudahGabriel/status/1748246465863205110
+     *
+     * This emits events to our iOS app to play the audio and set media session info.
+     * It also listens for events from the iOS app, such as iosaudiotimeupdate to indicate playback position.
+     */
+    export class IOSAudioPlayer implements PlatformAudio {
+        private _src: string | null = null;
+        private _currentTime = 0;
+        private _volume = 1;
+        private _duration = 0;
+        private readonly eventTarget = new EventTarget();
+        private readonly _isIOSWebApp: boolean;
+
+        public error: MediaError | null = null;
+
+        constructor() {
+            this._isIOSWebApp = navigator.userAgent.includes("Chavah iOS WKWebView");
+
+            if (this.isIOSWebApp) {
+                // Our iOS webview will send these events to use when their equivalents happen in native.
+                window.addEventListener("iosaudioended", () => this.dispatchEvent(new CustomEvent("ended")));
+                window.addEventListener("iosaudioerror", (e: ErrorEvent) => this.dispatchEvent(e));
+                window.addEventListener("iosaudiopause", () => this.dispatchEvent(new CustomEvent("pause")));
+                window.addEventListener("iosaudioplay", () => this.dispatchEvent(new CustomEvent("play")));
+                window.addEventListener("iosaudioplaying", () => this.dispatchEvent(new CustomEvent("playing")));
+                window.addEventListener("iosaudiowaiting", () => this.dispatchEvent(new CustomEvent("waiting")));
+                window.addEventListener("iosaudiostalled", () => this.dispatchEvent(new CustomEvent("stalled")));
+                window.addEventListener("iosaudiotimeupdate", (e: CustomEvent) => {
+                    this._currentTime = e.detail.currentTime;
+                    this._duration = e.detail.duration;
+                    this.dispatchEvent(new CustomEvent("timeupdate"));
+                });
+            }
+        }
+
+        get src(): string | null {
+            return this._src;
+        }
+
+        set src(val: string | null) {
+            this._src = val;
+            this.postMessageToNative("src", val);
+        }
+
+        get currentTime(): number {
+            return this._currentTime;
+        }
+
+        set currentTime(val: number) {
+            this._currentTime = val;
+            this.postMessageToNative("currentTime", val);
+        }
+
+        get volume(): number {
+            return this._volume;
+        }
+
+        set volume(val: number) {
+            this._volume = val;
+            this.postMessageToNative("volume", val);
+        }
+
+        get duration(): number {
+            return this._duration;
+        }
+
+        pause(): void {
+            this.postMessageToNative("pause");
+        }
+
+        play(): Promise<unknown> {
+            this.postMessageToNative("play");
+            return Promise.resolve();
+        }
+
+        load(): void {
+            // no op on iOS native audio.
+        }
+
+        addEventListener<K extends keyof HTMLMediaElementEventMap>(type: K, listener: (this: HTMLAudioElement, ev: HTMLMediaElementEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void {
+            this.eventTarget.addEventListener(type, listener, options);
+        }
+
+        /**
+         * Tells iOS web app to set media session info.
+         */
+        setMediaSession(album: string, artist: string, songTitle: string, artwork: string): void {
+            const args = JSON.stringify({
+                album,
+                artist,
+                songTitle,
+                artwork
+            });
+            this.postMessageToNative("mediasession", args);
+        }
+
+        get isIOSWebApp(): boolean {
+            return this._isIOSWebApp;
+        }
+
+        private dispatchEvent<K extends keyof HTMLMediaElementEventMap>(ev: HTMLMediaElementEventMap[K]): void {
+            this.eventTarget.dispatchEvent(ev);
+        }
+
+        private postMessageToNative(message: string, args?: string | number | null) {
+            const iosWebViewAudioHandler = this.isIOSWebApp && (window as any).webkit?.messageHandlers?.audiohandler;
+            if (iosWebViewAudioHandler) {
+                iosWebViewAudioHandler.postMessage({
+                    action: message,
+                    details: args
+                });
+            }
+        }
+    }
+
+    App.service("iosAudioPlayer", IOSAudioPlayer);
+}

Chavah.NetCore/wwwroot/js/Services/NavigatorMediaSessionService.ts

@@ -8,10 +8,13 @@
         private mediaSession: any | null;
 
         static $inject = [
-            "audioPlayer"
+            "audioPlayer",
+            "iosAudioPlayer"
         ];
 
-        constructor(private readonly audioPlayer: AudioPlayerService) {
+        constructor(
+            private readonly audioPlayer: AudioPlayerService,
+            private readonly iosAudioPlayer: IOSAudioPlayer) {
 
         }
 
@@ -108,9 +111,14 @@
                 } catch (error) {
                     console.log("unable to set mediaSession metadata", error);
                 }
+
+                // If we're in the iOS app, we may be using native audio. In that case, tell the iOS app about the new artwork.
+                if (this.iosAudioPlayer.isIOSWebApp && song) {
+                    this.iosAudioPlayer.setMediaSession(metadata.album, metadata.artist, metadata.title, metadata.artwork.length > 0 ? metadata.artwork[0].src : "");
+                }
             }
         }
     }
 
     App.service("navigatorMediaSession", NavigatorMediaSessionService);
-}
+}

Chavah.NetCore/wwwroot/js/Services/PlatformAudio.ts

@@ -0,0 +1,17 @@
+namespace BitShuva.Chavah {
+    /**
+     * Abstraction of HTML5 audio. On web and places that fully support HTML5 audio, it's implemented by actual HTML5 <audio> element.
+     * On platforms like iOS that have bugs or poor implementation of HTML5 audio, it is implemented by native audio on that platform.
+     */
+    export interface PlatformAudio {
+        src: string | null;
+        currentTime: number;
+        volume: number;
+        readonly duration: number;
+        error: MediaError | null;
+        pause(): void;
+        load(): void;
+        play(): Promise<unknown>;
+        addEventListener<K extends keyof HTMLMediaElementEventMap>(type: K, listener: (this: HTMLAudioElement, ev: HTMLMediaElementEventMap[K]) => any, options?: boolean | AddEventListenerOptions): void;
+    }
+}

Chavah.NetCore/wwwroot/views/partials/Header.html

@@ -143,7 +143,7 @@ <h3 ng-hide="vm.currentUserName">
                                 </a>
                             </li>
                             <li ng-hide="!vm.isAdmin">
-                                <a href="https://judahtemp.b-cdn.net/ios-webkit-audio-bug/bug.html">
+                                <a href="javascript:void(0)" ng-click="vm.skipToNearEnd()">
                                     <i class="fa fa-user-plus fa-fw"></i>
                                     Judah repro
                                 </a>