From b64841d4955fea66b29572e79c1047e6cad1224a Mon Sep 17 00:00:00 2001 From: Jan-Ivar Bruaroey Date: Wed, 22 Oct 2025 21:00:09 -0400 Subject: [PATCH 1/6] Clean up algorithms (removing main thread streams) --- index.bs | 184 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 128 insertions(+), 56 deletions(-) diff --git a/index.bs b/index.bs index 61e2649..8d80119 100644 --- a/index.bs +++ b/index.bs @@ -93,30 +93,31 @@ partial interface RTCRtpReceiver { }; +This allows manipulation of encoded frames in the media pipeline +between the processing steps of an {{RTCRtpSender}}'s underlying +encoder and packetizer, and/or +between an {{RTCRtpReceiver}}'s underlying depacketizer +and decoder. + ## Extension operation ## {#operation} At the time when a codec is initialized as part of the encoder, and the corresponding flag is set in the {{RTCPeerConnection}}'s {{RTCConfiguration}} argument, ensure that the codec is disabled and produces no output. - -### Stream creation ### {#stream-creation} - At construction of each {{RTCRtpSender}} or {{RTCRtpReceiver}}, run the following steps: 1. Initialize [=this=].`[[transform]]` to null. -1. Initialize [=this=].`[[readable]]` to a new {{ReadableStream}}. -1. Set up [=this=].`[[readable]]`. [=this=].`[[readable]]` is provided frames using the [$readEncodedData$] algorithm given |this| as parameter. -1. Initialize [=this=].`[[writable]]` to a new {{WritableStream}}. -1. Set up [=this=].`[[writable]]` with its [=WritableStream/set up/writeAlgorithm=] set to [$writeEncodedData$] given |this| as parameter and its [=WritableStream/set up/highWaterMark=] set to Infinity. -

highWaterMark is set to Infinity to explicitly disable backpressure.

1. Initialize [=this=].`[[pipeToController]]` to null. 1. Initialize [=this=].`[[lastReceivedFrameCounter]]` to 0. 1. Initialize [=this=].`[[lastEnqueuedFrameCounter]]` to 0. 1. [=Queue a task=] to run the following steps: 1. If [=this=].`[[pipeToController]]` is not null, abort these steps. 2. Set [=this=].`[[pipeToController]]` to a new {{AbortController}}. - - 3. Call pipeTo with [=this=].`[[readable]]`, [=this=].`[[writable]]`, preventClose equal to true, preventAbort equal to true, preventCancel equal to true and [=this=].`[[pipeToController]]`'s [=AbortController/signal=]. + +

TODO: Specify details of how + [=this=].`[[pipeToController]]` aborts streaming in the worker.

+ +### Stream processing ### {#stream-processing}

Streams backpressure can optimize throughput while limiting processing and memory consumption by pausing data production as early as possible in a data pipeline. @@ -127,8 +128,6 @@ The User Agent is responsible for doing these adaptations, especially since it c For those reasons, streams backpressure is disabled in WebRTC encoded transforms.

-### Stream processing ### {#stream-processing} - The readEncodedData algorithm is given a |rtcObject| as parameter. It is defined by running the following steps: 1. Wait for a frame to be produced by |rtcObject|'s encoder if it is a {{RTCRtpSender}} or |rtcObject|'s packetizer if it is a {{RTCRtpReceiver}}. 1. Increment |rtcObject|.`[[lastEnqueuedFrameCounter]]` by 1. @@ -345,7 +344,7 @@ The setEncryptionKey(|key|, |keyID|) met 4. Return |promise|. -# RTCRtpScriptTransform # {#scriptTransform} +# Script Transform # {#scriptTransform} In this section, the capture system refers to the system where media is sourced from and the sender system refers to the system that is sending RTP and RTCP packets to the receiver system where {{RTCEncodedFrameMetadata}} data is populated. @@ -906,65 +905,30 @@ Their [=deserialization steps=], given |serialized|, |value| and |realm|, are: 1. Set |value|'s metadata to the platform object representation of |serialized|.`[[metadata]]` 1. Set |value|.`[[data]]` to the [=sub-deserialization=] of |serialized|.`[[data]]`. -## Interfaces ## {#RTCRtpScriptTransformer-interfaces} -
-[Exposed=DedicatedWorker]
-interface RTCTransformEvent : Event {
-    readonly attribute RTCRtpScriptTransformer transformer;
-};
-
-partial interface DedicatedWorkerGlobalScope {
-    attribute EventHandler onrtctransform;
-};
-
-[Exposed=DedicatedWorker]
-interface RTCRtpScriptTransformer : EventTarget {
-    // Attributes and methods related to the transformer source
-    readonly attribute ReadableStream readable;
-    Promise<undefined> generateKeyFrame(optional DOMString rid);
-    Promise<undefined> sendKeyFrameRequest();
-    // Attributes and methods related to the transformer sink
-    readonly attribute WritableStream writable;
-    attribute EventHandler onkeyframerequest;
-    // Attributes for configuring the Javascript code
-    readonly attribute any options;
-};
 
+# RTCRtpScriptTransform interface # {#RTCRtpScriptTransform-interface}
+
 [Exposed=Window]
 interface RTCRtpScriptTransform {
     constructor(Worker worker, optional any options, optional sequence<object> transfer);
 };
-
-[Exposed=DedicatedWorker]
-interface KeyFrameRequestEvent : Event {
-  constructor(DOMString type, optional DOMString rid);
-  readonly attribute DOMString? rid;
-};
 

 
-## Operations ## {#RTCRtpScriptTransform-operations}
+## Constructor ## {#RTCRtpScriptTransform-constructor}
 
 The new RTCRtpScriptTransform(|worker|, |options|, |transfer|) constructor steps are:
-1. Set |t1| to an [=identity transform stream=].
-2. Set |t2| to an [=identity transform stream=].
-3. Set |this|.`[[writable]]` to |t1|.`[[writable]]`.
-4. Set |this|.`[[readable]]` to |t2|.`[[readable]]`.
 5. Let |serializedOptions| be the result of [$StructuredSerializeWithTransfer$](|options|, |transfer|).
-6. Let |serializedReadable| be the result of [$StructuredSerializeWithTransfer$](|t1|.`[[readable]]`, « |t1|.`[[readable]]` »).
-7. Let |serializedWritable| be the result of [$StructuredSerializeWithTransfer$](|t2|.`[[writable]]`, « |t2|.`[[writable]]` »).
 8. [=Queue a task=] on the DOM manipulation [=task source=] |worker|'s global scope to run the following steps:
     1. Let |transformerOptions| be the result of [$StructuredDeserializeWithTransfer$](|serializedOptions|, the current Realm).
-    2. Let |readable| be the result of [$StructuredDeserializeWithTransfer$](|serializedReadable|, the current Realm).
-    3. Let |writable| be the result of [$StructuredDeserializeWithTransfer$](|serializedWritable|, the current Realm).
-    4. Let |transformer| be a new {{RTCRtpScriptTransformer}}.
-    5. Set |transformer|.`[[options]]` to |transformerOptions|.
-    6. Set |transformer|.`[[readable]]` to |readable|.
-    7. Set |transformer|.`[[writable]]` to |writable|.
+    4. Let |transformer| be the result of [=RTCRtpScriptTransformer/creating=] a
+        {{RTCRtpScriptTransformer}} with |transformerOptions|.
     8. [=Fire an event=] named rtctransform using {{RTCTransformEvent}} with {{RTCTransformEvent/transformer}} set to |transformer| on |worker|’s global scope.
 
 // FIXME: Describe error handling (worker closing flag true at RTCRtpScriptTransform creation time. And worker being terminated while transform is processing data).
 
-Each RTCRtpScriptTransform has the following set of [$association steps$], given |rtcObject|:
+## Algorithms ## {#RTCRtpScriptTransform-algorithms}
+
+Each {{RTCRtpScriptTransform}} has the following set of [$association steps$], given |rtcObject|:
 1. Let |transform| be the {{RTCRtpScriptTransform}} object that owns the [$association steps$].
 1. Let |encoder| be |rtcObject|'s encoder if |rtcObject| is a {{RTCRtpSender}} or undefined otherwise.
 1. Let |depacketizer| be |rtcObject|'s depacketizer if |rtcObject| is a {{RTCRtpReceiver}} or undefined otherwise.
@@ -973,6 +937,97 @@ Each RTCRtpScriptTransform has the following set of [$association steps$], given
     1. Set |transformer|.`[[encoder]]` to |encoder|.
     1. Set |transformer|.`[[depacketizer]]` to |depacketizer|.
 
+# RTCRtpScriptTransformer interface # {#RTCRtpScriptTransformer-interface}
+
+[Exposed=DedicatedWorker]
+interface RTCRtpScriptTransformer : EventTarget {
+    // Attributes and methods related to the transformer source
+    readonly attribute ReadableStream readable;
+    Promise<undefined> generateKeyFrame(optional DOMString rid);
+    Promise<undefined> sendKeyFrameRequest();
+    // Attributes and methods related to the transformer sink
+    readonly attribute WritableStream writable;
+    attribute EventHandler onkeyframerequest;
+    // Attributes for configuring the Javascript code
+    readonly attribute any options;
+};
+

+
+## Internal slots ## {#RTCRtpScriptTransformer-internal-slots}
+
+A {{RTCRtpScriptTransformer}} object has the following internal slots.
+
+
+ 
+  
+   
+ 
+ 
+  
+   
+  
+   
+  
+   
+  
+   
+  
+   
+ 
+
Internal Slot
+   Description (non-normative)
+  
`[[depacketizer]]`
+   A [=depacketizer=] or undefined.
+  
`[[encoder]]`
+   An [=encoder=] or undefined.
+  
`[[options]]`
+   An optional {{Object}}, or null.
+  
`[[readable]]`
+   A {{ReadableStream}}.
+  
`[[writable]]`
+   A {{WritableStream}}.
+  

+
+

+To create an
+{{RTCRtpScriptTransformer}}, given an |options| object, perform the following steps:
+1. Let |transformer| be a [=new=] {{RTCRtpScriptTransformer}}, with:
+    : {{RTCRtpScriptTransformer/[[depacketizer]]}}
+    :: undefined
+    : {{RTCRtpScriptTransformer/[[encoder]]}}
+    :: undefined
+    : {{RTCRtpScriptTransformer/[[options]]}}
+    :: |options|
+    : {{RTCRtpScriptTransformer/[[readable]]}}
+    :: A new {{ReadableStream}}.
+    : {{RTCRtpScriptTransformer/[[writable]]}}
+    :: A new {{WritableStream}}.
+1. [=ReadableStream/Set up=] |transformer|.{{[[readable]]}}. 
The
+    [$readEncodedData$] algorithm, given |this| as parameter, provides encoded frames to it.
+    

+1. Let |writeAlgorithm| be an action that runs [$writeEncodedData$] with |this| as
+    parameter and |frame| as input, given |frame|.
+1. [=WritableStream/Set up=] |transformer|.{{[[writable]]}} with its
+    [=WritableStream/set up/writeAlgorithm=] set to |writeAlgorithm| and its [=WritableStream/set up/highWaterMark=] set to Infinity.
+    
highWaterMark is set to Infinity to explicitly disable backpressure.

+ 1. Return |transformer|.
+
+

+
+### Methods ### {#RTCRtpScriptTransformer-methods}
+

+    

+        getMetadata()
+    

+    

+        

+            Returns the metadata associated with the frame.
+        

+    

+

+
+## Operations ## {#RTCRtpScriptTransformer-operations}
+
 The generateKeyFrame(|rid|) method steps are:
 1. Let |promise| be a new promise.
 1. Run the [$generate key frame algorithm$] with |promise|, |this|.`[[encoder]]` and |rid|.
@@ -1004,6 +1059,23 @@ The onkeyframerequest EventHa
 
 ## Events ## {#RTCRtpScriptTransformer-events}
 
+
+[Exposed=DedicatedWorker]
+interface RTCTransformEvent : Event {
+    readonly attribute RTCRtpScriptTransformer transformer;
+};
+
+partial interface DedicatedWorkerGlobalScope {
+    attribute EventHandler onrtctransform;
+};
+
+[Exposed=DedicatedWorker]
+interface KeyFrameRequestEvent : Event {
+  constructor(DOMString type, optional DOMString rid);
+  readonly attribute DOMString? rid;
+};
+

+
 The following event fires on an {{RTCRtpScriptTransformer}}:
 
 * keyframerequest of type {{KeyFrameRequestEvent}} - fired when the sink determines that a key frame has been requested.

From 1dca7610768d7bffdee696695b0b75ef00e8a99b Mon Sep 17 00:00:00 2001
From: Jan-Ivar Bruaroey 
Date: Thu, 23 Oct 2025 20:28:47 -0400
Subject: [PATCH 2/6] Relocate more internal slots; add [[frameSource]] to
 simplify

---
 index.bs | 113 +++++++++++++++++++++++++++----------------------------
 1 file changed, 56 insertions(+), 57 deletions(-)

diff --git a/index.bs b/index.bs
index 8d80119..b771c20 100644
--- a/index.bs
+++ b/index.bs
@@ -95,9 +95,9 @@ partial interface RTCRtpReceiver {
 
 This allows manipulation of encoded frames in the media pipeline
 between the processing steps of an {{RTCRtpSender}}'s underlying
-encoder and packetizer, and/or
+encoder and packetizer, and/or
 between an {{RTCRtpReceiver}}'s underlying depacketizer
-and decoder.
+and decoder.
 
 ## Extension operation ## {#operation}
 
@@ -108,8 +108,6 @@ argument, ensure that the codec is disabled and produces no output.
 At construction of each {{RTCRtpSender}} or {{RTCRtpReceiver}}, run the following steps:
 1. Initialize [=this=].`[[transform]]` to null.
 1. Initialize [=this=].`[[pipeToController]]` to null.
-1. Initialize [=this=].`[[lastReceivedFrameCounter]]` to 0.
-1. Initialize [=this=].`[[lastEnqueuedFrameCounter]]` to 0.
 1. [=Queue a task=] to run the following steps:
     1. If [=this=].`[[pipeToController]]` is not null, abort these steps.
     2. Set [=this=].`[[pipeToController]]` to a new {{AbortController}}.
@@ -128,13 +126,14 @@ The User Agent is responsible for doing these adaptations, especially since it c
 For those reasons, streams backpressure is disabled in WebRTC encoded transforms.
 

 
-The readEncodedData algorithm is given a |rtcObject| as parameter. It is defined by running the following steps:
-1. Wait for a frame to be produced by |rtcObject|'s encoder if it is a {{RTCRtpSender}} or |rtcObject|'s packetizer if it is a {{RTCRtpReceiver}}.
-1. Increment |rtcObject|.`[[lastEnqueuedFrameCounter]]` by 1.
+The readEncodedData algorithm is given an {{RTCRtpScriptTransformer}}
+|transformer| as parameter. It is defined by running the following steps:
+1. Wait for a frame to be produced by |transformer|.{{[[frameSource]]}}.
+1. Increment |transformer|.`[[lastEnqueuedFrameCounter]]` by 1.
 1. Let |frame| be the newly produced frame.
-1. Set |frame|.`[[owner]]` to |rtcObject|.
-1. Set |frame|.`[[counter]]` to |rtcObject|.`[[lastEnqueuedFrameCounter]]`.
-1. If the frame has been produced by a {{RTCRtpReceiver}}:
+1. Set |frame|.`[[owner]]` to |transformer|.{{[[frameSource]]}}.
+1. Set |frame|.`[[counter]]` to |transformer|.{{[[lastEnqueuedFrameCounter]]}}.
+1. If |frame|.`[[owner]]` is a [=depacketizer=]:
     1. If the relevant RTP packet contains the
         [[RTP-EXT-CAPTURE-TIME|RTP Header Extension for Absolute Capture Time]], set |frame|.`[[captureTime]]` to the
         [[RTP-EXT-CAPTURE-TIME#absolute-capture-timestamp|absolute capture timestamp]] field and set |frame|.`[[senderCaptureTimeOffset]]`
@@ -145,30 +144,32 @@ The readEncodedData algorithm is given a |rtcObject| as p
         [[RTP-EXT-CAPTURE-TIME#timestamp-interpolation|timestamp interpolation]] and set |frame|.`[[senderCaptureTimeOffset]]`
         to the most recent value that was present.
     1. Otherwise, set |frame|.`[[captureTime]]` to undefined and set |frame|.`[[senderCaptureTimeOffset]]` to undefined.
-1. If the frame has been produced by a {{RTCRtpSender}}, set |frame|.`[[captureTime]]` to the capture timestamp
+1. If |frame|.`[[owner]]` is an [=encoder=], set |frame|.`[[captureTime]]` to the capture timestamp
     using the methodology described in [[RTP-EXT-CAPTURE-TIME#absolute-capture-timestamp]] and set frame.`[[senderCaptureTimeOffset]]`
     to undefined.
-1. [=ReadableStream/Enqueue=] |frame| in |rtcObject|.`[[readable]]`.
+1. [=ReadableStream/Enqueue=] |frame| into |transformer|.{{[[readable]]}}.
 
-The writeEncodedData algorithm is given a |rtcObject| as parameter and a |frame| as input. It is defined by running the following steps:
-1. If |frame|.`[[owner]]` is not equal to |rtcObject|, abort these steps and return [=a promise resolved with=] undefined. A processor cannot create frames, or move frames between streams.
-1. If |frame|.`[[counter]]` is equal or smaller than |rtcObject|.`[[lastReceivedFrameCounter]]`, abort these steps and return [=a promise resolved with=] undefined. A processor cannot reorder frames, although it may delay them or drop them.
-1. Set |rtcObject|.`[[lastReceivedFrameCounter]]` to |frame|`[[counter]]`.
+The writeEncodedData algorithm is given an {{RTCRtpScriptTransformer}}
+|transformer| as parameter and a |frame| as input. It is defined by running the following steps:
+1. If |frame|.`[[owner]]` is not equal to |transformer|.{{[[frameSource]]}}, abort these steps and return [=a promise resolved with=] undefined. A processor cannot create frames, or move frames between streams.
+1. If |frame|.`[[counter]]` is equal or smaller than |transformer|.`[[lastReceivedFrameCounter]]`, abort these steps and return [=a promise resolved with=] undefined. A processor cannot reorder frames, although it may delay them or drop them.
+1. Set |transformer|.`[[lastReceivedFrameCounter]]` to |frame|.`[[counter]]`.
 1. Let |data| be |frame|.`[[data]]`.
 1. Let |serializedFrame| be [$StructuredSerializeWithTransfer$](|frame|, « |data| »).
 1. Let |frameCopy| be [$StructuredDeserializeWithTransfer$](|serializedFrame|, |frame|'s [=relevant realm=]).
 1. Enqueue |frameCopy| for processing as if it came directly from the encoded data source, by running one of the following steps:
-    * If |rtcObject| is a {{RTCRtpSender}}, enqueue |frameCopy| to |rtcObject|'s packetizer, to be processed [=in parallel=].
-    * If |rtcObject| is a {{RTCRtpReceiver}}, enqueue |frameCopy| it to |rtcObject|'s decoder, to be processed [=in parallel=].
+    * If |transformer|.{{[[frameSource]]}} is an [=encoder=], enqueue |frameCopy| to its
+        associated [=packetizer=], to be processed [=in parallel=].
+    * If |transformer|.{{[[frameSource]]}} is a [=depacketizer=], enqueue |frameCopy| to its associated [=decoder=], to be processed [=in parallel=].
 1. Return [=a promise resolved with=] undefined.
 
-On sender side, as part of [$readEncodedData$], frames produced by |rtcObject|'s encoder MUST be enqueued in |rtcObject|.`[[readable]]` in the encoder's output order.
+On the sender side, as part of [$readEncodedData$], frames produced by the [=encoder=] MUST be [=ReadableStream/enqueued=] into |transformer|.{{[[readable]]}} in the [=encoder=]'s output order.
 As [$writeEncodedData$] ensures that the transform cannot reorder frames, the encoder's output order is also the order followed by packetizers to generate RTP packets and assign RTP packet sequence numbers.
-The packetizer may expect the transformed data to still conform to the original format, e.g. a series of NAL units separated by Annex B start codes.
+The [=packetizer=] may expect the transformed data to still conform to the original format, e.g. a series of NAL units separated by Annex B start codes.
 
-On receiver side, as part of [$readEncodedData$], frames produced by |rtcObject|'s packetizer MUST be enqueued in |rtcObject|.`[[readable]]` in the same encoder's output order.
-To ensure the order is respected, the depacketizer will typically use RTP packet sequence numbers to reorder RTP packets as needed before enqueuing frames in |rtcObject|.`[[readable]]`.
-As [$writeEncodedData$] ensures that the transform cannot reorder frames, this will be the order expected by |rtcObject|'s decoder.
+On the receiver side, as part of [$readEncodedData$], frames produced by the [=depacketizer=] MUST be [=ReadableStream/enqueued=] into |transformer|.{{[[readable]]}} in the same [=encoder=]'s output order.
+To ensure the order is respected, the [=depacketizer=] will typically use RTP packet sequence numbers to reorder RTP packets as needed before [=ReadableStream/enqueuing=] frames into |transformer|.{{[[readable]]}}.
+As [$writeEncodedData$] ensures that the transform cannot reorder frames, this will be the order expected by the [=decoder=].
 
 ## Extension attribute ## {#attribute}
 
@@ -202,7 +203,7 @@ The chain transform algorithm steps are defined as:
 1. If |newPipeToController|'s [=AbortController/signal=] is [=AbortSignal/aborted=], abort these steps.
 2. [=ReadableStreamDefaultReader/Release=] |reader|.
 3. [=WritableStreamDefaultWriter/Release=] |writer|.
-4. Assert that |newPipeToController| is the same object as |rtcObject|.`[[pipeToController]]`.
+4. [=Assert=] that |newPipeToController| is the same object as |rtcObject|.`[[pipeToController]]`.
 
 5. Call pipeTo with |rtcObject|.`[[readable]]`, |checkedTransform|.`[[writable]]`, preventClose equal to false, preventAbort equal to false, preventCancel equal to true and |newPipeToController|'s [=AbortController/signal=].
 6. Call pipeTo with |checkedTransform|.`[[readable]]`, |rtcObject|.`[[writable]]`, preventClose equal to true, preventAbort equal to true, preventCancel equal to false and |newPipeToController|'s [=AbortController/signal=].
@@ -930,12 +931,10 @@ The 
@@ -955,7 +954,7 @@ interface RTCRtpScriptTransformer : EventTarget {
 
 ## Internal slots ## {#RTCRtpScriptTransformer-internal-slots}
 
-A {{RTCRtpScriptTransformer}} object has the following internal slots.
+An {{RTCRtpScriptTransformer}} object has the following internal slots.
 
 
@@ -966,12 +965,8 @@ A {{RTCRtpScriptTransformer}} object has the following internal slots.
  
-   
-  
-   
+  
+   
+  
+   

  
  
   
`[[depacketizer]]`
-   A [=depacketizer=] or undefined.
-  
`[[encoder]]`
-   An [=encoder=] or undefined.
+   `[[frameSource]]`
+   An [=encoder=], a [=depacketizer=], or undefined.
   

   

    `[[options]]`
@@ -985,6 +980,14 @@ A {{RTCRtpScriptTransformer}} object has the following internal slots.
    `[[writable]]`
    A {{WritableStream}}.
   
`[[lastReceivedFrameCounter]]`
+   A count of frames received.
+  
`[[lastEnqueuedFrameCounter]]`
+   A count of frames enqueued.
+  

  
 

 
@@ -992,16 +995,18 @@ A {{RTCRtpScriptTransformer}} object has the following internal slots.
 To create an
 {{RTCRtpScriptTransformer}}, given an |options| object, perform the following steps:
 1. Let |transformer| be a [=new=] {{RTCRtpScriptTransformer}}, with:
-    : {{RTCRtpScriptTransformer/[[depacketizer]]}}
-    :: undefined
-    : {{RTCRtpScriptTransformer/[[encoder]]}}
+    : {{RTCRtpScriptTransformer/[[frameSource]]}}
     :: undefined
     : {{RTCRtpScriptTransformer/[[options]]}}
     :: |options|
     : {{RTCRtpScriptTransformer/[[readable]]}}
-    :: A new {{ReadableStream}}.
+    :: A new {{ReadableStream}}
     : {{RTCRtpScriptTransformer/[[writable]]}}
-    :: A new {{WritableStream}}.
+    :: A new {{WritableStream}}
+    : {{RTCRtpScriptTransformer/[[lastReceivedFrameCounter]]}}
+    :: 0
+    : {{RTCRtpScriptTransformer/[[lastEnqueuedFrameCounter]]}}
+    :: 0
 1. [=ReadableStream/Set up=] |transformer|.{{[[readable]]}}. 
The
     [$readEncodedData$] algorithm, given |this| as parameter, provides encoded frames to it.
     

@@ -1030,20 +1035,16 @@ To create an
 
 The generateKeyFrame(|rid|) method steps are:
 1. Let |promise| be a new promise.
-1. Run the [$generate key frame algorithm$] with |promise|, |this|.`[[encoder]]` and |rid|.
+1. Run the [$generate key frame algorithm$] with |promise|, |this|.`[[frameSource]]` and |rid|.
 1. Return |promise|.
 
 The sendKeyFrameRequest() method steps are:
 1. Let |promise| be a new promise.
-1. Run the [$send request key frame algorithm$] with |promise| and |this|.`[[depacketizer]]`.
+1. Run the [$send request key frame algorithm$] with |promise| and |this|.`[[frameSource]]`.
 1. Return |promise|.
 
 ## Attributes ## {#RTCRtpScriptTransformer-attributes}
 
-A {{RTCRtpScriptTransformer}} has the following private slots called `[[depacketizer]]`, `[[encoder]]`, `[[options]]`, `[[readable]]` and `[[writable]]`.
-In addition, a {{RTCRtpScriptTransformer}} is always associated with its parent {{RTCRtpScriptTransform}} transform.
-This allows algorithms to go from an {{RTCRtpScriptTransformer}} object to its {{RTCRtpScriptTransform}} parent and vice versa.
-
 The options getter steps are:
 1. Return [=this=].`[[options]]`.
 
@@ -1082,21 +1083,19 @@ The following event fires on an {{RTCRtpScriptTransformer}}:
 
 The steps that generate an event of type {{KeyFrameRequestEvent}} are as follows:
 
-Given a {{RTCRtpScriptTransformer}} |transform|:
-
-When |transform|'s `[[encoder]]` receives a keyframe request, for instance from an incoming RTCP Picture Loss Indication (PLI)
+When the [=encoder=] of an associated {{RTCRtpScriptTransformer}} |transformer| receives a keyframe request, for instance from an incoming RTCP Picture Loss Indication (PLI)
 or Full Intra Refresh (FIR), queue a task to perform the following steps:
 
 1. Set |rid| to the RID of the appropriate layer, or undefined if the request is not for a specific layer.
-1. [=Fire an event=] named `keyframerequest` at |transform| using {{KeyFrameRequestEvent}} with its {{Event/cancelable}} attribute initialized to "true", and with {{KeyFrameRequestEvent/rid}} set to |rid|.
+1. [=Fire an event=] named `keyframerequest` at |transformer| using {{KeyFrameRequestEvent}} with its {{Event/cancelable}} attribute initialized to "true", and with {{KeyFrameRequestEvent/rid}} set to |rid|.
 1. If the event's [=Event/canceled flag=] is true, abort these steps.
-1. Run the [$generate key frame algorithm$] with a new promise, |transform|.`[[encoder]]` and |rid|.
+1. Run the [$generate key frame algorithm$] with a new promise, |transformer|.`[[frameSource]]` and |rid|.
 
 ## KeyFrame Algorithms ## {#KeyFrame-algorithms}
 
 The generate key frame algorithm, given |promise|, |encoder| and |rid|, is defined by running these steps:
-1. If |encoder| is undefined, reject |promise| with {{InvalidStateError}}, abort these steps.
-1. If |encoder| does not belong to a video {{RTCRtpReceiver}}, reject |promise| with {{InvalidStateError}}, abort these steps.
+1. If |encoder| is not an [=encoder=], reject |promise| with {{InvalidStateError}}, abort these steps.
+1. If |encoder| does not belong to a video {{RTCRtpSender}}, reject |promise| with {{InvalidStateError}}, abort these steps.
 1. If |rid| is defined, but does not conform to the grammar requirements specified
     in Section 10 of [[!RFC8851]], then reject |promise| with {{TypeError}} and abort
     these steps.
@@ -1116,9 +1115,9 @@ The generate key frame algorithm, given |promise|, |encod
     1. For each |layer| in |layers| (if any), instruct |encoder| to generate a key frame for its
         next provided video frame to that |layer|.
 
-For any {{RTCRtpScriptTransformer}} named |transformer|, the following steps are run just before any |frame| is enqueued in |transformer|.`[[readable]]`:
-1. Let |encoder| be |transformer|.`[[encoder]]`.
-1. If |encoder| or |encoder|.`[[pendingKeyFrameTasks]]` is undefined, abort these steps.
+For any [=encoder=] associated with an {{RTCRtpScriptTransformer}} |transformer|, the user agent MUST run the following steps just before any |frame| is [=ReadableStream/enqueued=] into |transformer|.{{[[readable]]}}:
+1. Let |encoder| be |transformer|.{{[[frameSource]]}}.
+1. If |encoder|.`[[pendingKeyFrameTasks]]` is undefined, abort these steps.
 1. If |frame| is not a video {{RTCEncodedVideoFrameType/"key"}} frame, abort these steps.
 1. For each |task| in |encoder|.`[[pendingKeyFrameTasks]]`, run the following steps:
     1. If |frame| was generated for a layer [=list/contained=] in |task|.`[[layers]]`,
@@ -1133,7 +1132,7 @@ the resolution callbacks of the promises are always executed just before the cor
 If the promise is associated with several layers, it will be resolved once key frames have been enqueued for all of them.
 
 The send request key frame algorithm, given |promise| and |depacketizer|, is defined by running these steps:
-1. If |depacketizer| is undefined, reject |promise| with {{InvalidStateError}}, abort these steps.
+1. If |depacketizer| is not a [=depacketizer=], reject |promise| with {{InvalidStateError}}, abort these steps.
 1. If |depacketizer| does not belong to a video {{RTCRtpReceiver}}, reject |promise| with {{InvalidStateError}}, abort these steps.
 1. [=In parallel=], run the following steps:
     1. If sending a Full Intra Request (FIR) by |depacketizer|'s receiver is not deemed appropriate, [=resolve=] |promise| with undefined and abort these steps.

From cddd9c9e031d1cd9b2e96d7bacef2b71700d3da4 Mon Sep 17 00:00:00 2001
From: Jan-Ivar Bruaroey 
Date: Thu, 23 Oct 2025 21:01:34 -0400
Subject: [PATCH 3/6] fix method snafu

---
 index.bs | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/index.bs b/index.bs
index b771c20..b3877d8 100644
--- a/index.bs
+++ b/index.bs
@@ -1019,19 +1019,7 @@ To create an
 
 
 
-### Methods ### {#RTCRtpScriptTransformer-methods}
-

-    

-        getMetadata()
-    

-    

-        

-            Returns the metadata associated with the frame.
-        

-    

-

-
-## Operations ## {#RTCRtpScriptTransformer-operations}
+## Methods ## {#RTCRtpScriptTransformer-methods}
 
 The generateKeyFrame(|rid|) method steps are:
 1. Let |promise| be a new promise.

From 48b9e6cf57d063ac25661f649b7d1acc84cb5cd7 Mon Sep 17 00:00:00 2001
From: Jan-Ivar Bruaroey 
Date: Fri, 24 Oct 2025 09:35:21 -0400
Subject: [PATCH 4/6] Fix numbering upset in two places

---
 index.bs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/index.bs b/index.bs
index b3877d8..44987bc 100644
--- a/index.bs
+++ b/index.bs
@@ -918,12 +918,12 @@ interface RTCRtpScriptTransform {
 ## Constructor ## {#RTCRtpScriptTransform-constructor}
 
 The new RTCRtpScriptTransform(|worker|, |options|, |transfer|) constructor steps are:
-5. Let |serializedOptions| be the result of [$StructuredSerializeWithTransfer$](|options|, |transfer|).
-8. [=Queue a task=] on the DOM manipulation [=task source=] |worker|'s global scope to run the following steps:
+1. Let |serializedOptions| be the result of [$StructuredSerializeWithTransfer$](|options|, |transfer|).
+1. [=Queue a task=] on the DOM manipulation [=task source=] |worker|'s global scope to run the following steps:
     1. Let |transformerOptions| be the result of [$StructuredDeserializeWithTransfer$](|serializedOptions|, the current Realm).
-    4. Let |transformer| be the result of [=RTCRtpScriptTransformer/creating=] a
+    1. Let |transformer| be the result of [=RTCRtpScriptTransformer/creating=] a
         {{RTCRtpScriptTransformer}} with |transformerOptions|.
-    8. [=Fire an event=] named rtctransform using {{RTCTransformEvent}} with {{RTCTransformEvent/transformer}} set to |transformer| on |worker|’s global scope.
+    1. [=Fire an event=] named rtctransform using {{RTCTransformEvent}} with {{RTCTransformEvent/transformer}} set to |transformer| on |worker|’s global scope.
 
 // FIXME: Describe error handling (worker closing flag true at RTCRtpScriptTransform creation time. And worker being terminated while transform is processing data).
 

From a991ba0bef915009a85dd3041041e5afc83cdd47 Mon Sep 17 00:00:00 2001
From: Jan-Ivar Bruaroey 
Date: Sun, 26 Oct 2025 09:38:36 -0400
Subject: [PATCH 5/6] Address the common algorithms (transform setter,
 RTCRtpTransform)

---
 index.bs | 63 ++++++++++++++++++++++++++++++--------------------------
 1 file changed, 34 insertions(+), 29 deletions(-)

diff --git a/index.bs b/index.bs
index 44987bc..13a4568 100644
--- a/index.bs
+++ b/index.bs
@@ -93,27 +93,39 @@ partial interface RTCRtpReceiver {
 };
-This allows manipulation of encoded frames in the media pipeline -between the processing steps of an {{RTCRtpSender}}'s underlying -encoder and packetizer, and/or +This allows manipulation of encoded frames in the media +pipeline between the processing steps of an {{RTCRtpSender}}'s +underlying encoder and packetizer, and/or between an {{RTCRtpReceiver}}'s underlying depacketizer and decoder. -## Extension operation ## {#operation} +The [=encoder=] and [=depacketizer=] each have a +[[transformFrameAlgorithm]] internal slot initialized to +the passthrough algorithm which, given an [=encoded frame=] +|frame|, is to return |frame|. + +Whenever the [=encoder=] outputs a [=encoded frame|frame=], the user +agent MUST invoke the [=encoder=].[=[[transformFrameAlgorithm]]=] with +it and pass the result to the associated [=packetizer=] in place of +the original frame. -At the time when a codec is initialized as part of the encoder, and the -corresponding flag is set in the {{RTCPeerConnection}}'s {{RTCConfiguration}} -argument, ensure that the codec is disabled and produces no output. +Whenever the [=depacketizer=] outputs a [=encoded frame|frame=], the +user agent MUST invoke the [=depacketizer=].[=[[transformFrameAlgorithm]]=] +with it and pass the result to the associated [=decoder=] in place of +the original frame. + +## Extension operation ## {#operation} At construction of each {{RTCRtpSender}} or {{RTCRtpReceiver}}, run the following steps: 1. Initialize [=this=].`[[transform]]` to null. 1. Initialize [=this=].`[[pipeToController]]` to null. +1. Initialize [=this=].`[[frameSource]]` to [=this=]'s [=encoder=] if [=this=] is an {{RTCRtpSender}} or [=this=]'s [=depacketizer=] otherwise. 1. [=Queue a task=] to run the following steps: 1. If [=this=].`[[pipeToController]]` is not null, abort these steps. - 2. Set [=this=].`[[pipeToController]]` to a new {{AbortController}}. - -

TODO: Specify details of how - [=this=].`[[pipeToController]]` aborts streaming in the worker.

+ 1. Set [=this=].`[[pipeToController]]` to a new {{AbortController}}. + 1. Let |frameSource| be [=this=].`[[frameSource]]`. + 1. [=In parallel=], set |frameSource|.[=[[transformFrameAlgorithm]]=] to the + [=passthrough algorithm=]. ### Stream processing ### {#stream-processing} @@ -173,9 +185,7 @@ As [$writeEncodedData$] ensures that the transform cannot reorder frames, this w ## Extension attribute ## {#attribute} -A RTCRtpTransform has private slots: -* `[[readable]]` of type {{ReadableStream}}. -* `[[writable]]` of type {{WritableStream}}. +An RTCRtpTransform has a private slot: * `[[owner]]` of type {{RTCRtpSender}} or {{RTCRtpReceiver}}, initialized to null. Each RTCRtpTransform has an association steps set, which is empty by default. @@ -185,28 +195,23 @@ The transform getter step The `transform` setter steps are: 1. Let |transform| be the argument to the setter. -1. Let |checkedTransform| set to |transform| if it is not null or to an [=identity transform stream=] otherwise. -1. If |checkedTransform|.`[[owner]]` is not null, throw a {{InvalidStateError}} and abort these steps. -1. Let |reader| be the result of [=ReadableStream/getting a reader=] for |checkedTransform|.`[[readable]]`. -1. Let |writer| be the result of [=WritableStream/getting a writer=] for |checkedTransform|.`[[writable]]`. -1. Set |checkedTransform|.`[[owner]]` to [=this=]. +1. If |transform| is not null and |transform|.`[[owner]]` is not null, throw a {{InvalidStateError}} and abort these steps. +1. Set |transform|.`[[owner]]` to [=this=]. 1. Initialize |newPipeToController| to a new {{AbortController}}. 1. If [=this=].`[[pipeToController]]` is not null, run the following steps: - 1. [=AbortSignal/Add=] the [$chain transform algorithm$] to [=this=].`[[pipeToController]]`'s [=AbortController/signal=]. + 1. [=AbortSignal/Add=] the [$apply transform$] to [=this=].`[[pipeToController]]`'s [=AbortController/signal=]. 2. [=AbortController/signal abort=] on [=this=].`[[pipeToController]]`. -1. Else, run the [$chain transform algorithm$] steps. +1. Else, run the [$apply transform$] steps. 1. Set [=this=].`[[pipeToController]]` to |newPipeToController|. 1. Set [=this=].`[[transform]]` to |transform|. 1. Run the steps in the set of [$association steps$] of |transform| with [=this=]. -The chain transform algorithm steps are defined as: +The apply transform steps are defined as: 1. If |newPipeToController|'s [=AbortController/signal=] is [=AbortSignal/aborted=], abort these steps. -2. [=ReadableStreamDefaultReader/Release=] |reader|. -3. [=WritableStreamDefaultWriter/Release=] |writer|. -4. [=Assert=] that |newPipeToController| is the same object as |rtcObject|.`[[pipeToController]]`. - -5. Call pipeTo with |rtcObject|.`[[readable]]`, |checkedTransform|.`[[writable]]`, preventClose equal to false, preventAbort equal to false, preventCancel equal to true and |newPipeToController|'s [=AbortController/signal=]. -6. Call pipeTo with |checkedTransform|.`[[readable]]`, |rtcObject|.`[[writable]]`, preventClose equal to true, preventAbort equal to true, preventCancel equal to false and |newPipeToController|'s [=AbortController/signal=]. +1. [=Assert=] that |newPipeToController| is the same object as |rtcObject|.`[[pipeToController]]`. + +

TODO: Specify details of how + [=this=].`[[pipeToController]]` replaces the transform in the media pipeline.

This algorithm is defined so that transforms can be updated dynamically. There is no guarantee on which frame will happen the switch from the previous transform to the new transform. @@ -931,7 +936,7 @@ The transform getter steps are: -1. Return [=this=].`[[transform]]`. +## Extension attribute ## {#attribute} -The `transform` setter steps are: +The transform getter steps are to +return [=this=].`[[transform]]`. The setter steps are: 1. Let |transform| be the argument to the setter. 1. If |transform| is not null and |transform|.`[[owner]]` is not null, throw a {{InvalidStateError}} and abort these steps. 1. Set |transform|.`[[owner]]` to [=this=]. -1. Initialize |newPipeToController| to a new {{AbortController}}. -1. If [=this=].`[[pipeToController]]` is not null, run the following steps: - 1. [=AbortSignal/Add=] the [$apply transform$] to [=this=].`[[pipeToController]]`'s [=AbortController/signal=]. - 2. [=AbortController/signal abort=] on [=this=].`[[pipeToController]]`. -1. Else, run the [$apply transform$] steps. -1. Set [=this=].`[[pipeToController]]` to |newPipeToController|. +1. Let |oldTransform| be [=this=].`[[transform]]`. +1. If |oldTransform| is not null, run |oldTransform|'s [$disassociation algorithm$]. 1. Set [=this=].`[[transform]]` to |transform|. -1. Run the steps in the set of [$association steps$] of |transform| with [=this=]. - -The apply transform steps are defined as: -1. If |newPipeToController|'s [=AbortController/signal=] is [=AbortSignal/aborted=], abort these steps. -1. [=Assert=] that |newPipeToController| is the same object as |rtcObject|.`[[pipeToController]]`. - -

TODO: Specify details of how - [=this=].`[[pipeToController]]` replaces the transform in the media pipeline.

+1. If |transform| is not null, run |transform|'s [$association algorithm$] with [=this=]. +1. If |transform| is null, run the following steps: + 1. Let |frameSource| be [=this=].`[[frameSource]]`. + 1. [=In parallel=], set |frameSource|.[=[[transformFrameAlgorithm]]=] to the [=passthrough algorithm=]. This algorithm is defined so that transforms can be updated dynamically. There is no guarantee on which frame will happen the switch from the previous transform to the new transform. @@ -920,26 +904,69 @@ interface RTCRtpScriptTransform { }; +## Internal slots ## {#RTCRtpScriptTransform-internal-slots} + +An {{RTCRtpScriptTransform}} object has the following internal slot: + + + + + + + + + + +
Internal Slot + Description (non-normative) +
`[[worker]]` + The {{Worker}} provided in the constructor. +
+ ## Constructor ## {#RTCRtpScriptTransform-constructor} The new RTCRtpScriptTransform(|worker|, |options|, |transfer|) constructor steps are: +1. Initialize [=this=]'s internal slot as follows: + : {{RTCRtpScriptTransform/[[worker]]}} + :: |worker| 1. Let |serializedOptions| be the result of [$StructuredSerializeWithTransfer$](|options|, |transfer|). -1. [=Queue a task=] on the DOM manipulation [=task source=] |worker|'s global scope to run the following steps: +1. [=Queue a global task=] on the DOM manipulation [=task source=] with |worker|'s + {{WorkerGlobalScope}} to run the following steps: 1. Let |transformerOptions| be the result of [$StructuredDeserializeWithTransfer$](|serializedOptions|, the current Realm). 1. Let |transformer| be the result of [=RTCRtpScriptTransformer/creating=] a {{RTCRtpScriptTransformer}} with |transformerOptions|. - 1. [=Fire an event=] named rtctransform using {{RTCTransformEvent}} with {{RTCTransformEvent/transformer}} set to |transformer| on |worker|’s global scope. + 1. [=Fire an event=] named rtctransform using {{RTCTransformEvent}} with {{RTCTransformEvent/transformer}} set to |transformer| on |transformer|’s [=relevant global object=]. // FIXME: Describe error handling (worker closing flag true at RTCRtpScriptTransform creation time. And worker being terminated while transform is processing data). ## Algorithms ## {#RTCRtpScriptTransform-algorithms} -Each {{RTCRtpScriptTransform}} has the following set of [$association steps$], given |rtcObject|: -1. Let |transform| be the {{RTCRtpScriptTransform}} object that owns the [$association steps$]. -1. Let |frameSource| be |rtcObject|'s `[[frameSource=]]`. -1. [=Queue a task=] on the DOM manipulation [=task source=] |worker|'s global scope to run the following steps: - 1. Let |transformer| be the {{RTCRtpScriptTransformer}} object associated to |transform|. +Each {{RTCRtpScriptTransform}} has the following [$association algorithm$], given |rtcObject|: +1. Let |transform| be the {{RTCRtpScriptTransform}} object that owns the [$association algorithm$]. +1. Let |frameSource| be |rtcObject|'s `[[frameSource]]`. +1. Let |workerGlobalScope| be |transform|.{{[[worker]]}}'s {{WorkerGlobalScope}}. +1. [=Queue a global task=] on the DOM manipulation [=task source=] with |workerGlobalScope| + to run the following steps: + 1. Let |transformer| be the {{RTCRtpScriptTransformer}} object associated with |transform|. 1. Set |transformer|.{{[[frameSource]]}} to |frameSource|. +1. [=In parallel=], set |frameSource|.[=[[transformFrameAlgorithm]]=] to the following steps, + given an [=encoded frame=] |frame| as input: + 1. [=Queue a global task=] on the DOM manipulation [=task source=] with |workerGlobalScope| + to run the following steps: + 1. Let |transformer| be the {{RTCRtpScriptTransformer}} object associated with |transform|. + 1. Let |jsFrame| be a new {{RTCEncodedVideoFrame}} from |frame| if |frame| is a video + frame, or a new {{RTCEncodedAudioFrame}} from |frame| otherwise. + 1. Invoke [$readEncodedData$] with |transformer| and |jsFrame|. + 1. Wait for |frameSource|.[=[[processedFramesQueue]]=] to become non-empty. + 1. Return the result of [=dequeueing=] from |frameSource|.[=[[processedFramesQueue]]=]. + +Each {{RTCRtpScriptTransform}} has the following [$disassociation algorithm$]: +1. Let |transform| be the {{RTCRtpScriptTransform}} object that owns the [$disassociation algorithm$]. +1. [=Queue a global task=] on the DOM manipulation [=task source=] with + |transform|.{{[[worker]]}}'s {{WorkerGlobalScope}} to run the following steps: + 1. Let |transformer| be the {{RTCRtpScriptTransformer}} object associated with |transform|. + 1. [=ReadableStream/cancel=] |transformer|.{{[[readable]]}}. + 1. [=WritableStream/abort=] |transformer|.{{[[writable]]}}. # RTCRtpScriptTransformer interface # {#RTCRtpScriptTransformer-interface} 
@@ -959,7 +986,7 @@ interface RTCRtpScriptTransformer : EventTarget {
 
 ## Internal slots ## {#RTCRtpScriptTransformer-internal-slots}
 
-An {{RTCRtpScriptTransformer}} object has the following internal slots.
+An {{RTCRtpScriptTransformer}} object has the following internal slots: