Add feedback on video connection

src/components/widgets/VideoPlayer.vue

@@ -3,12 +3,27 @@
     <div v-if="nameSelectedStream === undefined" class="no-video-alert">
       <span>No video stream selected.</span>
     </div>
-    <div v-else-if="!namesAvailableStreams.includes(nameSelectedStream)" class="no-video-alert">
+    <div
+      v-else-if="!namesAvailableStreams.isEmpty() && !namesAvailableStreams.includes(nameSelectedStream)"
+      class="no-video-alert"
+    >
       <p>The selected stream is not available.</p>
       <p>Please check its source or select another stream.</p>
     </div>
-    <div v-else-if="mediaStream === undefined" class="no-video-alert">
-      <span>Loading stream...</span>
+    <div v-else-if="!streamConnected" class="no-video-alert">
+      <div class="no-video-alert">
+        <p>
+          <span class="text-xl font-bold">Server status: </span>
+          <span>{{ serverStatus }}</span>
+        </p>
+        <p>
+          <span class="text-xl font-bold">Stream status: </span>
+          <span>{{ streamStatus }}</span>
+        </p>
+      </div>
+    </div>
+    <div v-else class="no-video-alert">
+      <p>Loading stream...</p>
     </div>
     <video ref="videoElement" muted autoplay playsinline disablePictureInPicture>
       Your browser does not support the video tag.
@@ -90,6 +105,7 @@ const widget = toRefs(props).widget
 const nameSelectedStream = ref<string | undefined>()
 const videoElement = ref<HTMLVideoElement | undefined>()
 const mediaStream = ref<MediaStream | undefined>()
+const streamConnected = ref(false)
 
 onBeforeMount(() => {
   // Set the default initial values that are not present in the widget options
@@ -123,6 +139,11 @@ const streamConnectionRoutine = setInterval(() => {
   if (!isEqual(updatedMediaStream, mediaStream.value)) {
     mediaStream.value = updatedMediaStream
   }
+
+  const updatedStreamState = videoStore.getStreamData(widget.value.options.streamName)?.connected ?? false
+  if (updatedStreamState !== streamConnected.value) {
+    streamConnected.value = updatedStreamState
+  }
 }, 1000)
 onBeforeUnmount(() => clearInterval(streamConnectionRoutine))
 
@@ -158,6 +179,16 @@ const rotateStyle = computed(() => {
 const transformStyle = computed(() => {
   return `${flipStyle.value} ${rotateStyle.value}`
 })
+
+const serverStatus = computed(() => {
+  if (nameSelectedStream.value === undefined) return 'Unknown.'
+  return videoStore.getStreamData(nameSelectedStream.value)?.webRtcManager.signallerStatus ?? 'Unknown.'
+})
+
+const streamStatus = computed(() => {
+  if (nameSelectedStream.value === undefined) return 'Unknown.'
+  return videoStore.getStreamData(nameSelectedStream.value)?.webRtcManager.streamStatus ?? 'Unknown.'
+})
 </script>
 
 <style scoped>

src/composables/webRTC.ts

@@ -11,18 +11,14 @@ import type { Stream } from '@/libs/webrtc/signalling_protocol'
  *
  */
 interface startStreamReturn {
-  /**
-   * A list of Available WebRTC streams from Mavlink Camera Manager to be chosen from
-   */
-  availableStreams: Ref<Array<Stream>>
-  /**
-   * A list of IPs from WebRTC candidates that are available
-   */
-  availableICEIPs: Ref<Array<string>>
   /**
    * MediaStream object, if WebRTC stream is chosen
    */
   mediaStream: Ref<MediaStream | undefined>
+  /**
+   * Connection state
+   */
+  connected: Ref<boolean>
   /**
    * Current status of the signalling
    */
@@ -37,11 +33,12 @@ interface startStreamReturn {
  *
  */
 export class WebRTCManager {
-  private availableStreams: Ref<Array<Stream>> = ref(new Array<Stream>())
+  public availableStreams: Ref<Array<Stream>> = ref(new Array<Stream>())
   public availableICEIPs: Ref<Array<string>> = ref(new Array<string>())
   private mediaStream: Ref<MediaStream | undefined> = ref()
-  private signallerStatus: Ref<string> = ref('waiting...')
-  private streamStatus: Ref<string> = ref('waiting...')
+  public signallerStatus: Ref<string> = ref('waiting...')
+  public streamStatus: Ref<string> = ref('waiting...')
+  private connected = ref(false)
   private consumerId: string | undefined
   private streamName: string | undefined
   private session: Session | undefined
@@ -126,9 +123,8 @@ export class WebRTCManager {
     })
 
     return {
-      availableStreams: this.availableStreams,
-      availableICEIPs: this.availableICEIPs,
       mediaStream: this.mediaStream,
+      connected: this.connected,
       signallerStatus: this.signallerStatus,
       streamStatus: this.streamStatus,
     }
@@ -161,12 +157,9 @@ export class WebRTCManager {
     this.hasEnded = false
     // Requests a new consumer ID
     if (this.consumerId === undefined) {
-      this.signaller.requestConsumerId(
-        (newConsumerId: string): void => {
-          this.consumerId = newConsumerId
-        },
-        (newStatus: string): void => this.updateStreamStatus(newStatus)
-      )
+      this.signaller.requestConsumerId((newConsumerId: string): void => {
+        this.consumerId = newConsumerId
+      })
     }
 
     this.availableStreams.value = []
@@ -189,10 +182,6 @@ export class WebRTCManager {
 
     // Asks for available streams, which will trigger the consumer "onAvailableStreams" callback
     window.setTimeout(() => {
-      if (!this.waitingForAvailableStreamsAnswer) {
-        return
-      }
-
       // Register the parser to update the list of streams when the signaller receives the answer
       this.signaller.parseAvailableStreamsAnswer((availableStreams): void => {
         if (!this.waitingForAvailableStreamsAnswer) {
@@ -234,6 +223,13 @@ export class WebRTCManager {
     console.debug('Capabilities:', event.track.getCapabilities?.())
   }
 
+  /**
+   * Called when a peer is connected
+   */
+  private onPeerConnected(): void {
+    this.connected.value = true
+  }
+
   /**
    *
    * @param {Stream} stream
@@ -243,14 +239,9 @@ export class WebRTCManager {
     console.debug(`[WebRTC] Requesting stream:`, stream)
 
     // Requests a new Session ID
-    this.signaller.requestSessionId(
-      consumerId,
-      stream.id,
-      (receivedSessionId: string): void => {
-        this.onSessionIdReceived(stream, stream.id, receivedSessionId)
-      },
-      (newStatus: string): void => this.updateStreamStatus(newStatus)
-    )
+    this.signaller.requestSessionId(consumerId, stream.id, (receivedSessionId: string): void => {
+      this.onSessionIdReceived(stream, stream.id, receivedSessionId)
+    })
 
     this.hasEnded = false
   }
@@ -331,31 +322,26 @@ export class WebRTCManager {
       this.rtcConfiguration,
       this.selectedICEIPs,
       (event: RTCTrackEvent): void => this.onTrackAdded(event),
+      (): void => this.onPeerConnected(),
       (availableICEIPs: string[]) => (this.availableICEIPs.value = availableICEIPs),
-      (_sessionId, reason) => this.onSessionClosed(reason)
+      (_sessionId, reason) => this.onSessionClosed(reason),
+      (status: string): void => this.updateStreamStatus(status)
     )
 
     // Registers Session callback for the Signaller endSession parser
-    this.signaller.parseEndSessionQuestion(
-      this.consumerId!,
-      producerId,
-      this.session.id,
-      (sessionId, reason) => {
-        console.debug(`[WebRTC] Session ${sessionId} ended. Reason: ${reason}`)
-        this.session = undefined
-        this.hasEnded = true
-      },
-      (newStatus: string): void => this.updateSignallerStatus(newStatus)
-    )
+    this.signaller.parseEndSessionQuestion(this.consumerId!, producerId, this.session.id, (sessionId, reason) => {
+      console.debug(`[WebRTC] Session ${sessionId} ended. Reason: ${reason}`)
+      this.session = undefined
+      this.hasEnded = true
+    })
 
     // Registers Session callbacks for the Signaller Negotiation parser
     this.signaller.parseNegotiation(
       this.consumerId!,
       producerId,
       this.session.id,
       this.session.onIncomingICE.bind(this.session),
-      this.session.onIncomingSDP.bind(this.session),
-      (newStatus: string): void => this.updateSignallerStatus(newStatus)
+      this.session.onIncomingSDP.bind(this.session)
     )
 
     const msg = `Session ${this.session.id} successfully started`

src/libs/webrtc/session.ts

@@ -5,6 +5,8 @@ import type { Stream } from '@/libs/webrtc/signalling_protocol'
 type OnCloseCallback = (sessionId: string, reason: string) => void
 type OnTrackAddedCallback = (event: RTCTrackEvent) => void
 type onNewIceRemoteAddressCallback = (availableICEIPs: string[]) => void
+type OnStatusChangeCallback = (status: string) => void
+type OnPeerConnectedCallback = () => void
 
 /**
  * An abstraction for the Mavlink Camera Manager WebRTC Session
@@ -21,8 +23,10 @@ export class Session {
   private selectedICEIPs: string[]
   public rtcConfiguration: RTCConfiguration
   public onTrackAdded?: OnTrackAddedCallback
+  public onPeerConnected?: OnPeerConnectedCallback
   public onNewIceRemoteAddress?: onNewIceRemoteAddressCallback
   public onClose?: OnCloseCallback
+  public onStatusChange?: OnStatusChangeCallback
 
   /**
    * Creates a new Session instance, connecting with a given Stream
@@ -33,8 +37,10 @@ export class Session {
    * @param {RTCConfiguration} rtcConfiguration - Configuration for the RTC connection, such as Turn and Stun servers
    * @param {string[]} selectedICEIPs - A whitelist for ICE IP addresses, ignored if empty
    * @param {OnTrackAddedCallback} onTrackAdded - An optional callback for when a track is added to this session
+   * @param {OnPeerConnectedCallback} onPeerConnected - An optional callback for when the peer is connected
    * @param {onNewIceRemoteAddressCallback} onNewIceRemoteAddress - An optional callback for when a new ICE candidate IP addres is available
    * @param {OnCloseCallback} onClose - An optional callback for when this session closes
+   * @param {OnStatusChangeCallback} onStatusChange - An optional callback for internal status change
    */
   constructor(
     sessionId: string,
@@ -44,15 +50,19 @@ export class Session {
     rtcConfiguration: RTCConfiguration,
     selectedICEIPs: string[] = [],
     onTrackAdded?: OnTrackAddedCallback,
+    onPeerConnected?: OnPeerConnectedCallback,
     onNewIceRemoteAddress?: onNewIceRemoteAddressCallback,
-    onClose?: OnCloseCallback
+    onClose?: OnCloseCallback,
+    onStatusChange?: OnStatusChangeCallback
   ) {
     this.id = sessionId
     this.consumerId = consumerId
     this.stream = stream
     this.onTrackAdded = onTrackAdded
+    this.onPeerConnected = onPeerConnected
     this.onNewIceRemoteAddress = onNewIceRemoteAddress
     this.onClose = onClose
+    this.onStatusChange = onStatusChange
     this.status = ''
     this.signaller = signaller
     this.rtcConfiguration = rtcConfiguration
@@ -195,16 +205,22 @@ export class Session {
       !this.selectedICEIPs.isEmpty() &&
       !this.selectedICEIPs.some((address) => candidate.candidate!.includes(address))
     ) {
+      this.onStatusChange?.(`Ignoring ICE candidate ${candidate.candidate}`)
       console.debug(`[WebRTC] [Session] ICE candidate ignored: ${JSON.stringify(candidate, null, 4)}`)
       return
     }
 
     this.peerConnection
       .addIceCandidate(candidate)
-      .then(() => console.debug(`[WebRTC] [Session] ICE candidate added: ${JSON.stringify(candidate, null, 4)}`))
-      .catch((reason) =>
+      .then(() => {
+        const msg = `ICE candidate added.`
+        console.debug(`[WebRTC] [Session] ${msg} ${JSON.stringify(candidate, null, 4)}`)
+        this.onStatusChange?.(msg)
+      })
+      .catch((reason) => {
         console.error(`[WebRTC] [Session] Failed adding ICE candidate ${candidate}. Reason: ${reason}`)
-      )
+        this.onStatusChange?.(`Failed adding ICE candidate ${candidate}. Reason: ${reason}`)
+      })
   }
 
   /**
@@ -225,7 +241,9 @@ export class Session {
    */
   private onIceCandidateError(event: Event): void {
     const ev = event as RTCPeerConnectionIceErrorEvent
-    console.debug(`[WebRTC] [Session] ICE Candidate "${ev.url}" negotiation failed`)
+    const msg = `ICE Candidate "${ev.url}" negotiation failed.`
+    console.debug(`[WebRTC] [Session] ${msg}`)
+    this.onStatusChange?.(msg)
   }
 
   /**
@@ -242,7 +260,9 @@ export class Session {
    */
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   private onNegotiationNeeded(_event: Event): void {
-    console.debug('[WebRTC] [Session] Peer Connection is waiting for negotiation...')
+    const msg = 'Peer Connection is waiting for negotiation...'
+    console.debug(msg)
+    this.onStatusChange?.(msg)
   }
 
   /**
@@ -251,6 +271,7 @@ export class Session {
   private onIceConnectionStateChange(): void {
     const msg = `ICEConnection state changed to "${this.peerConnection.iceConnectionState}"`
     console.debug('[WebRTC] [Session] ' + msg)
+    this.onStatusChange?.(msg)
 
     if (this.peerConnection.iceConnectionState === 'failed') {
       this.peerConnection.restartIce()
@@ -263,6 +284,11 @@ export class Session {
   private onConnectionStateChange(): void {
     const msg = `RTCPeerConnection state changed to "${this.peerConnection.connectionState}"`
     console.debug('[WebRTC] [Session] ' + msg)
+    this.onStatusChange?.(msg)
+
+    if (this.peerConnection.connectionState === 'connected') {
+      this.onPeerConnected?.()
+    }
 
     if (this.peerConnection.connectionState === 'failed') {
       this.onClose?.(this.id, 'PeerConnection failed')
@@ -283,7 +309,9 @@ export class Session {
    */
   private onIceGatheringStateChange(): void {
     if (this.peerConnection.iceGatheringState === 'complete') {
-      console.debug(`[WebRTC] [Session] ICE gathering completed for session ${this.id}`)
+      const msg = `ICE gathering completed for session ${this.id}`
+      console.debug(`[WebRTC] [Session] ${msg}`)
+      this.onStatusChange?.(msg)
     }
   }