Expose Frame Packet Trailer (FrameMetadata) for video

Runtime/Scripts/Participant.cs

@@ -1001,4 +1001,21 @@ public LocalDataTrack Track
 
         public PublishDataTrackError Error { get; private set; }
     }
+
+    /// Helpers for setting <see cref="TrackPublishOptions"/> fields whose underlying type
+    /// lives in Google.Protobuf (e.g. RepeatedField&lt;T&gt;). Unity's default Assembly-CSharp
+    /// does not auto-reference Google.Protobuf, so callers without an asmdef cannot use a
+    /// collection initializer or call <c>.Add</c> on the repeated field directly. These
+    /// helpers keep Google.Protobuf types out of the caller's signature.
+    public static class TrackPublishOptionsExtensions
+    {
+        public static TrackPublishOptions WithPacketTrailerFeatures(
+            this TrackPublishOptions options,
+            params PacketTrailerFeature[] features)
+        {
+            foreach (var feature in features)
+                options.PacketTrailerFeatures.Add(feature);
+            return options;
+        }
+    }
 }

Runtime/Scripts/RtcVideoSource.cs

@@ -30,6 +30,12 @@ public enum VideoStreamSource
         /// Called when we receive a new texture (first texture or the resolution changed)
         public event TextureReceiveDelegate TextureReceived;
 
+        public delegate FrameMetadata FrameMetadataDelegate();
+        /// Invoked once per outgoing frame. Return null (default) to send no trailer.
+        /// To actually serialize the trailer onto RTP, also enable the matching
+        /// PacketTrailerFeatures on the TrackPublishOptions used at publish time.
+        public FrameMetadataDelegate MetadataProvider { get; set; }
+
         protected Texture2D _previewTexture;
         protected NativeArray<byte> _captureBuffer;
         protected VideoStreamSource _sourceType;
@@ -175,6 +181,8 @@ protected virtual bool SendFrame()
                 var now = DateTimeOffset.UtcNow;
                 capture.TimestampUs = now.ToUnixTimeMilliseconds() * 1000 + (now.Ticks % TimeSpan.TicksPerMillisecond) / 10;
                 capture.Buffer = buffer;
+                var metadata = MetadataProvider?.Invoke();
+                if (metadata != null) capture.Metadata = metadata;
                 using var response = request.Send();
                 _reading = false;
                 _requestPending = false;

Runtime/Scripts/VideoFrame.cs

@@ -12,12 +12,14 @@ public sealed class VideoFrame
 
         public long Timestamp;
         public VideoRotation Rotation;
+        public FrameMetadata Metadata;
 
-        public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation)
+        public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation, FrameMetadata metadata = null)
         {
             _info = info;
             Timestamp = timeStamp;
             Rotation = rotation;
+            Metadata = metadata;
         }
     }
 

Runtime/Scripts/VideoStream.cs

@@ -242,7 +242,7 @@ private void OnVideoStreamEvent(VideoStreamEvent e)
             // Avoid allocating VideoFrame objects when nobody is observing them.
             if (FrameReceived != null)
             {
-                var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation);
+                var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation, e.FrameReceived.Metadata);
                 FrameReceived.Invoke(frame);
             }
         }

Tests/PlayMode/TrackTests.cs

@@ -149,7 +149,7 @@ public IEnumerator RemoteTrackPublication_SetVideoQuality_DoesNotThrow()
             var publisherRoom = context.Rooms[0];
             var subscriberRoom = context.Rooms[1];
 
-            var videoSource = new StubVideoSource();
+            var videoSource = new TestVideoSource();
             var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom);
 
             // Video track uses a stub source that never pushes frames. TrackSubscribed may not
@@ -283,10 +283,10 @@ public IEnumerator RemoteTrackPublication_PublisherDisablesCamera_UpdatesFlagAnd
             var publisherRoom = context.Rooms[0];
             var subscriberRoom = context.Rooms[1];
 
-            var videoSource = new StubVideoSource();
+            var videoSource = new TestVideoSource();
             var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom);
 
-            // StubVideoSource never pushes frames, so TrackSubscribed may not fire on the
+            // TestVideoSource (pushFrames=false) never pushes frames, so TrackSubscribed may not fire on the
             // subscriber. The RemoteTrackPublication still propagates via TrackPublished.
             var publishedExp = new Expectation(timeoutSeconds: 10f);
             subscriberRoom.TrackPublished += (_, _) => publishedExp.Fulfill();

Tests/PlayMode/Utils/CoroutineRunner.cs

@@ -0,0 +1,12 @@
+using UnityEngine;
+
+namespace LiveKit.PlayModeTests.Utils
+{
+    /// <summary>
+    /// Empty MonoBehaviour used by tests to host long-running coroutines
+    /// (e.g. <see cref="RtcVideoSource.Update"/> and <see cref="VideoStream.Update"/>)
+    /// that the test itself cannot host because <c>[UnityTest]</c> bodies must yield
+    /// back to the test runner.
+    /// </summary>
+    public class CoroutineRunner : MonoBehaviour { }
+}

Tests/PlayMode/Utils/TestVideoSource.cs

@@ -0,0 +1,53 @@
+using LiveKit.Proto;
+using Unity.Collections;
+
+namespace LiveKit.PlayModeTests.Utils
+{
+    /// <summary>
+    /// Test-only <see cref="RtcVideoSource"/> registered with FFI at a fixed
+    /// resolution. Two modes via <paramref name="pushFrames"/>:
+    /// <list type="bullet">
+    /// <item><c>false</c> (default): never pushes frames. Use when the test only
+    /// needs the publication to propagate via signaling (e.g.
+    /// <see cref="RemoteTrackPublication"/> APIs that operate on metadata).</item>
+    /// <item><c>true</c>: pushes a continuous stream of zero-filled RGBA frames.
+    /// Use when the test needs actual media flow (e.g. validating per-frame
+    /// metadata round-trips through the FFI / RTP path).</item>
+    /// </list>
+    /// </summary>
+    public sealed class TestVideoSource : RtcVideoSource
+    {
+        private readonly int _width;
+        private readonly int _height;
+        private readonly bool _pushFrames;
+
+        public override int GetWidth() => _width;
+        public override int GetHeight() => _height;
+
+        protected override VideoRotation GetVideoRotation() => VideoRotation._0;
+
+        protected override bool ReadBuffer()
+        {
+            if (!_pushFrames) return false;
+
+            if (!_captureBuffer.IsCreated)
+            {
+                _captureBuffer = new NativeArray<byte>(
+                    _width * _height * 4,
+                    Allocator.Persistent,
+                    NativeArrayOptions.ClearMemory);
+            }
+            _requestPending = true;
+            return false;
+        }
+
+        public TestVideoSource(bool pushFrames = false, int width = 16, int height = 16)
+            : base(VideoStreamSource.Texture, VideoBufferType.Rgba)
+        {
+            _width = width;
+            _height = height;
+            _pushFrames = pushFrames;
+            Init();
+        }
+    }
+}

Tests/PlayMode/VideoFrameMetadataTests.cs

@@ -0,0 +1,103 @@
+using System.Collections;
+using LiveKit.PlayModeTests.Utils;
+using LiveKit.Proto;
+using NUnit.Framework;
+using UnityEngine;
+using UnityEngine.TestTools;
+
+namespace LiveKit.PlayModeTests
+{
+    public class VideoFrameMetadataTests
+    {
+        const string VideoTrackName = "metadata-video-track";
+
+        static (TestRoomContext.ConnectionOptions publisher, TestRoomContext.ConnectionOptions subscriber) TwoPeers()
+        {
+            var publisher = TestRoomContext.ConnectionOptions.Default;
+            publisher.Identity = "metadata-publisher";
+            var subscriber = TestRoomContext.ConnectionOptions.Default;
+            subscriber.Identity = "metadata-subscriber";
+            return (publisher, subscriber);
+        }
+
+        [UnityTest, Category("E2E")]
+        public IEnumerator VideoFrame_AttachedMetadata_ReceivedOnSubscriber()
+        {
+            var (publisher, subscriber) = TwoPeers();
+            using var context = new TestRoomContext(new[] { publisher, subscriber });
+            yield return context.ConnectAll();
+            Assert.IsNull(context.ConnectionError, context.ConnectionError);
+
+            var publisherRoom = context.Rooms[0];
+            var subscriberRoom = context.Rooms[1];
+
+            const uint expectedFrameId = 42u;
+            const ulong expectedUserTs = 0x1122334455667788UL;
+
+            var source = new TestVideoSource(pushFrames: true);
+            source.MetadataProvider = () => new FrameMetadata
+            {
+                FrameId = expectedFrameId,
+                UserTimestamp = expectedUserTs,
+            };
+
+            var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, source, publisherRoom);
+
+            var subscribedExp = new Expectation(timeoutSeconds: 10f);
+            RemoteVideoTrack receivedRemoteTrack = null;
+            subscriberRoom.TrackSubscribed += (track, _, _) =>
+            {
+                if (track is RemoteVideoTrack rv)
+                {
+                    receivedRemoteTrack = rv;
+                    subscribedExp.Fulfill();
+                }
+            };
+
+            var options = new TrackPublishOptions
+            {
+                Source = TrackSource.SourceCamera,
+            }.WithPacketTrailerFeatures(
+                PacketTrailerFeature.PtfUserTimestamp,
+                PacketTrailerFeature.PtfFrameId);
+            var pub = publisherRoom.LocalParticipant.PublishTrack(localTrack, options);
+            yield return pub;
+            Assert.IsFalse(pub.IsError);
+
+            // Host the source's Update coroutine on a throwaway MonoBehaviour so the
+            // test body can yield on Expectations without owning the producer loop.
+            var runnerObj = new GameObject("metadata-test-runner");
+            var runner = runnerObj.AddComponent<CoroutineRunner>();
+            source.Start();
+            runner.StartCoroutine(source.Update());
+
+            yield return subscribedExp.Wait();
+            Assert.IsNull(subscribedExp.Error);
+            Assert.IsNotNull(receivedRemoteTrack);
+
+            var stream = new VideoStream(receivedRemoteTrack);
+            var metadataExp = new Expectation(timeoutSeconds: 10f);
+            FrameMetadata receivedMetadata = null;
+            stream.FrameReceived += frame =>
+            {
+                if (receivedMetadata != null || frame.Metadata == null) return;
+                receivedMetadata = frame.Metadata;
+                metadataExp.Fulfill();
+            };
+            stream.Start();
+            runner.StartCoroutine(stream.Update());
+
+            yield return metadataExp.Wait();
+            Assert.IsNull(metadataExp.Error, "expected a frame with metadata within timeout");
+            Assert.IsNotNull(receivedMetadata);
+            Assert.AreEqual(expectedFrameId, receivedMetadata.FrameId, "frame_id mismatch");
+            Assert.AreEqual(expectedUserTs, receivedMetadata.UserTimestamp, "user_timestamp mismatch");
+
+            source.Stop();
+            stream.Stop();
+            stream.Dispose();
+            source.Dispose();
+            Object.Destroy(runnerObj);
+        }
+    }
+}