Concurrent encoding pipeline

compositor-module/compositor-proxy-api.yaml

@@ -86,12 +86,26 @@ paths:
             schema:
               description: Describes when the key frame should be generated.
               type: object
+              required:
+                - bufferId
+                - bufferCreationSerial
+                - bufferContentSerial
               properties:
-                syncSerial:
+                bufferId:
+                  type: number
+                  format: int32
+                  minimum: 1
+                  description: The buffer id.
+                bufferContentSerial:
                   type: number
                   format: int32
-                  minimum: 0
-                  description: An optional sync serial. When provided the encoded frame will be send immediately.
+                  minimum: 1
+                  description: The buffer content serial.
+                bufferCreationSerial:
+                  type: number
+                  format: int32
+                  minimum: 1
+                  description: The buffer creation serial.
       responses:
         202:
           description: Key frame successfully requested.
@@ -104,7 +118,7 @@ paths:
               schema:
                 type: string
         404:
-          description: client or surface not found.
+          description: client, surface or buffer not found.
           content:
             text/plain:
               schema:

compositor-module/demo-compositor/src/index.ts

@@ -5,6 +5,8 @@ import {
   initWasm,
 } from '../../src'
 
+const proxyHost = 'localhost'
+
 async function main() {
   // load web assembly libraries
   await initWasm()
@@ -15,8 +17,8 @@ async function main() {
 
   // Get an HTML5 canvas for use as an output for the compositor. Multiple outputs can be used.
   const canvas: HTMLCanvasElement = document.createElement('canvas')
-  canvas.width = 1440
-  canvas.height = 900
+  canvas.width = 1920
+  canvas.height = 1080
   canvas.style.width = `${canvas.width}`
   canvas.style.height = `${canvas.height}`
 
@@ -44,19 +46,19 @@ async function main() {
   const compositorProxyConnector = createCompositorProxyConnector(session, remoteSocket)
 
   const connect8081Button: HTMLButtonElement = document.createElement('button')
-  connect8081Button.textContent = `connect to ws://localhost:8081?compositorSessionId=${compositorSessionId}`
+  connect8081Button.textContent = `connect to ws://${proxyHost}:8081?compositorSessionId=${compositorSessionId}`
 
   connect8081Button.onclick = () => {
-    const compositorProxyURL = new URL('ws://localhost:8081')
+    const compositorProxyURL = new URL(`ws://${proxyHost}:8081`)
     compositorProxyURL.searchParams.append('compositorSessionId', compositorSessionId)
     compositorProxyConnector.connectTo(compositorProxyURL)
   }
 
   const connect8082Button: HTMLButtonElement = document.createElement('button')
-  connect8082Button.textContent = `connect to ws://localhost:8082?compositorSessionId=${compositorSessionId}`
+  connect8082Button.textContent = `connect to ws://${proxyHost}:8082?compositorSessionId=${compositorSessionId}`
 
   connect8082Button.onclick = () => {
-    const compositorProxyURL = new URL('ws://localhost:8082')
+    const compositorProxyURL = new URL(`ws://${proxyHost}:8082`)
     compositorProxyURL.searchParams.append('compositorSessionId', compositorSessionId)
     compositorProxyConnector.connectTo(compositorProxyURL)
   }

compositor-module/src/BufferContents.ts

@@ -23,5 +23,5 @@ export default interface BufferContents<T> {
   readonly size: Size
   readonly mimeType: 'video/h264' | 'image/png' | 'image/rgba' | 'image/canvas'
   readonly pixelContent: T
-  readonly serial: number
+  readonly contentSerial: number
 }

compositor-module/src/RemoteAppLauncher.ts

@@ -33,12 +33,12 @@ export default class RemoteAppLauncher implements CompositorProxyConnector {
     this.remoteSocket = remoteSocket
   }
 
-  async connectTo(appEndpointURL: URL): Promise<Client> {
-    this.remoteSocket.ensureXWayland(appEndpointURL)
+  async connectTo(compositorProxyURL: URL): Promise<Client> {
+    this.remoteSocket.ensureXWayland(compositorProxyURL)
     const clientId = randomString()
     return this.remoteSocket.onWebSocket(
-      createRetransmittingWebSocket(appEndpointURL, clientId),
-      appEndpointURL,
+      createRetransmittingWebSocket(compositorProxyURL, clientId),
+      compositorProxyURL,
       clientId,
     )
   }

compositor-module/src/RemoteOutOfBandChannel.ts

@@ -20,7 +20,6 @@ import Session from './Session'
 export const enum RemoteOutOfBandListenOpcode {
   BufferSentStarted = 1,
   BufferCreation = 2,
-  BufferContents = 3,
   WebSocketCreationRequest = 5,
   RecycledResourceIds = 6,
   XWMConnectionRequest = 7,

compositor-module/src/RemoteSocket.ts

@@ -22,7 +22,7 @@ import RemoteOutOfBandChannel, {
   RemoteOutOfBandListenOpcode,
   RemoteOutOfBandSendOpcode,
 } from './RemoteOutOfBandChannel'
-import StreamingBuffer from './remotestreaming/StreamingBuffer'
+import { StreamingBuffer } from './remotestreaming/StreamingBuffer'
 import Session from './Session'
 import XWaylandShell from './xwayland/XWaylandShell'
 import { XWindowManager } from './xwayland/XWindowManager'
@@ -31,6 +31,7 @@ import { createRemoteWebFS } from './WebFS'
 import { Configuration, EncoderApi } from './api'
 import Surface from './Surface'
 import { createClientEncodersFeedback } from './remotestreaming/EncoderFeedback'
+import { deliverContentToBufferStream } from './remotestreaming/BufferStream'
 
 const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' as const
 
@@ -183,11 +184,27 @@ class RemoteSocket {
           clientEncodersFeedback: createClientEncodersFeedback(clientId, encoderApi),
         }
 
+        this.setupFrameDataChannel(client, compositorProxyURL, clientId)
         resolve(client)
       })
     })
   }
 
+  private setupFrameDataChannel(client: Client, compositorProxyURL: URL, connectionId: string) {
+    const url = new URL(compositorProxyURL)
+    url.searchParams.append('frameData', '')
+    const frameDataChannel = createRetransmittingWebSocket(url, connectionId)
+
+    frameDataChannel.addEventListener('message', (message) => {
+      if (client.connection.closed) {
+        return
+      }
+
+      const messageData = message.data as ArrayBuffer
+      deliverContentToBufferStream(client, messageData)
+    })
+  }
+
   private setupClientOutOfBandHandlers(
     webSocket: WebSocketLike,
     client: Client,
@@ -214,30 +231,11 @@ class RemoteSocket {
         return
       }
 
-      const wlBufferResource = new WlBufferResource(client, new Uint32Array(message.buffer, message.byteOffset)[0], 1)
-      wlBufferResource.implementation = StreamingBuffer.create(wlBufferResource)
-    })
-
-    // listen for buffer contents arriving. opcode: 3
-    outOfBandChannel.setListener(RemoteOutOfBandListenOpcode.BufferContents, (outOfBandMessage) => {
-      if (client.connection.closed) {
-        return
-      }
-
-      const bufferContentsDataView = new DataView(outOfBandMessage.buffer, outOfBandMessage.byteOffset)
-      const bufferId = bufferContentsDataView.getUint32(0, true)
-      const wlBufferResource = client.connection.wlObjects[bufferId] as WlBufferResource
-
-      // Buffer might be destroyed while waiting for it's content to arrive.
-      if (wlBufferResource) {
-        const streamingBuffer = wlBufferResource.implementation as StreamingBuffer
-
-        const bufferContents = new Uint8Array(
-          outOfBandMessage.buffer,
-          outOfBandMessage.byteOffset + Uint32Array.BYTES_PER_ELEMENT,
-        )
-        streamingBuffer.bufferStream.onBufferContents(bufferContents)
-      }
+      const payload = new Uint32Array(message.buffer, message.byteOffset)
+      const resourceId = payload[0]
+      const creationSerial = payload[1]
+      const wlBufferResource = new WlBufferResource(client, resourceId, 1)
+      wlBufferResource.implementation = StreamingBuffer.create(wlBufferResource, creationSerial)
     })
 
     // listen for web socket creation request. opcode: 5

compositor-module/src/Session.ts

@@ -100,8 +100,6 @@ async function webVideoDecoderConfig(): Promise<VideoDecoderConfig | undefined>
 
     const softwareDecoderSupport = await VideoDecoder.isConfigSupported(softwareDecoderConfig)
     if (softwareDecoderSupport) {
-      // Software decoding often has worse performance then our WASM+WebGL decoder, so we mark it as unsupported.
-      // return undefined
       return softwareDecoderConfig
     }
   }

compositor-module/src/Surface.ts

@@ -59,7 +59,7 @@ export interface SurfaceState {
   bufferTransform: number
   bufferScale: number
 
-  bufferResourceDestroyListener: () => void
+  readonly bufferResourceDestroyListener: () => void
   buffer?: WlBufferResource
   bufferContents?: BufferContents<unknown>
 
@@ -308,18 +308,18 @@ class Surface implements WlSurfaceRequests {
       | undefined
     if (bufferImplementation && this.pendingState.bufferContents === undefined) {
       try {
-        this.session.logger.trace(`|- Awaiting buffer contents with serial: ${serial ?? 'NO SERIAL'}`)
-        const startBufferContents = Date.now()
-        this.pendingState.bufferContents = await bufferImplementation.getContents(this, serial)
-
-        this.session.logger.trace(
-          `|--> Buffer contents with serial: ${serial ?? 'NO SERIAL'} took ${Date.now() - startBufferContents}ms`,
-        )
+        // console.log(`|- Awaiting buffer contents with serial: ${serial ?? 'NO SERIAL'}`)
+        // const startBufferContents = Date.now()
+        const bufferContents = bufferImplementation.getContents(this, serial)
+        this.pendingState.bufferContents = bufferContents instanceof Promise ? await bufferContents : bufferContents
+        // console.log(
+        //   `|--> Buffer contents with serial: ${serial ?? 'NO SERIAL'} took ${Date.now() - startBufferContents}ms`,
+        // )
         if (this.destroyed) {
           return
         }
       } catch (e: any) {
-        this.session.logger.warn(`[surface: ${resource.id}] - Failed to receive buffer contents.`, e.toString())
+        this.session.logger.warn(`[surface: ${resource.id}] - Failed to process buffer contents.`, e.toString())
       }
     }
     if (this.encoderFeedback && serial !== undefined) {

compositor-module/src/XdgToplevel.ts

@@ -131,26 +131,25 @@ export default class XdgToplevel implements XdgToplevelRequests, DesktopSurfaceR
       return
     }
 
-    const geometry = this.xdgSurface.surface.geometry
-
-    if (this.next.state.maximized) {
-      if (this.next.size.width !== geometry.size.width || this.next.size.width !== geometry.size.width) {
-        const errorMessage = `Client protocol error. xdg_surface buffer (${geometry.size.width}x${geometry.size.height}) does not match the configured maximum state (${this.next.size.width}x${this.next.size.height})`
-        this.session.logger.warn(errorMessage)
-        this.resource.postError(XdgWmBaseError.invalidSurfaceState, errorMessage)
-        return
-      }
-    }
-
-    if (
-      this.next.state.fullscreen &&
-      (this.next.size.width !== geometry.size.width || this.next.size.width !== geometry.size.width)
-    ) {
-      const errorMessage = `Client protocol error. xdg_surface buffer (${geometry.size.width}x${geometry.size.height})  is larger than the configured fullscreen state  (${this.next.size.width}x${this.next.size.height})`
-      this.session.logger.warn(errorMessage)
-      this.resource.postError(XdgWmBaseError.invalidSurfaceState, errorMessage)
-      return
-    }
+    // const geometry = this.xdgSurface.surface.geometry
+    // if (this.next.state.maximized) {
+    //   if (this.next.size.width !== geometry.size.width || this.next.size.width !== geometry.size.width) {
+    //     const errorMessage = `Client protocol error. xdg_surface buffer (${geometry.size.width}x${geometry.size.height}) does not match the configured maximum state (${this.next.size.width}x${this.next.size.height})`
+    //     this.session.logger.warn(errorMessage)
+    //     this.resource.postError(XdgWmBaseError.invalidSurfaceState, errorMessage)
+    //     return
+    //   }
+    // }
+    //
+    // if (
+    //   this.next.state.fullscreen &&
+    //   (this.next.size.width !== geometry.size.width || this.next.size.width !== geometry.size.width)
+    // ) {
+    //   const errorMessage = `Client protocol error. xdg_surface buffer (${geometry.size.width}x${geometry.size.height})  is larger than the configured fullscreen state  (${this.next.size.width}x${this.next.size.height})`
+    //   this.session.logger.warn(errorMessage)
+    //   this.resource.postError(XdgWmBaseError.invalidSurfaceState, errorMessage)
+    //   return
+    // }
 
     this.current.state = { ...this.next.state }
     this.current.minSize = this.next.minSize

compositor-module/src/browser/selection.ts

@@ -8,16 +8,20 @@ import { initBrowserTextSelection } from './text-selection'
 // *and* it also doesn't tell us which mimetypes it supports
 // *and* each browser supports different mimetypes,
 // so we just slim it down to the lowest common set of mimetypes... :(
-const allowedMimeTypes = ['text/plain', 'text/html', 'image/png']
+const browserMimeTypes = ['text/plain', 'text/html', 'image/png']
 
 let browserOffers: ClipboardItems | undefined
 let browserOffersTotalSize = 0
 
-async function blobFromDataSource(mimeType: string, dataSource: DataSource): Promise<Blob> {
+async function blobFromDataSource(
+  browserMimeType: string,
+  dataSourceMimeType: string,
+  dataSource: DataSource,
+): Promise<Blob> {
   const [readFD, writeFD] = await dataSource.webfs.mkfifo()
-  dataSource.send(mimeType, writeFD)
+  dataSource.send(dataSourceMimeType, writeFD)
   const dataBlob = await readFD.readBlob()
-  return new Blob([dataBlob], { type: mimeType })
+  return new Blob([dataBlob], { type: browserMimeType })
 }
 
 function handleWaylandDataSourceUpdate(seat: Seat) {
@@ -28,16 +32,19 @@ function handleWaylandDataSourceUpdate(seat: Seat) {
     }
 
     const clipboardDataEntries = dataSource.mimeTypes
-      .map((mimeType) => {
-        for (const allowedMimeType of allowedMimeTypes) {
-          if (mimeType.indexOf(allowedMimeType) !== -1) {
-            return allowedMimeType
+      .map((dataSourceMimeType) => {
+        for (const browserMimeType of browserMimeTypes) {
+          if (dataSourceMimeType.indexOf(browserMimeType) !== -1) {
+            return [browserMimeType, dataSourceMimeType] as const
           }
         }
-        return mimeType
+        return [dataSourceMimeType, dataSourceMimeType] as const
       })
-      .filter((mimeType) => allowedMimeTypes.includes(mimeType))
-      .map((mimeType) => [mimeType, blobFromDataSource(mimeType, dataSource)])
+      .filter(([browserMimeType]) => browserMimeTypes.includes(browserMimeType))
+      .map(
+        ([browserMimeType, dataSourceMimeType]) =>
+          [browserMimeType, blobFromDataSource(browserMimeType, dataSourceMimeType, dataSource)] as const,
+      )
     if (clipboardDataEntries.length === 0) {
       navigator.clipboard.writeText('')
     } else {