diff --git a/Runtime/Scripts/Participant.cs b/Runtime/Scripts/Participant.cs index bc49710b..4da028c0 100644 --- a/Runtime/Scripts/Participant.cs +++ b/Runtime/Scripts/Participant.cs @@ -1001,4 +1001,21 @@ public LocalDataTrack Track public PublishDataTrackError Error { get; private set; } } + + /// Helpers for setting fields whose underlying type + /// lives in Google.Protobuf (e.g. RepeatedField<T>). Unity's default Assembly-CSharp + /// does not auto-reference Google.Protobuf, so callers without an asmdef cannot use a + /// collection initializer or call .Add on the repeated field directly. These + /// helpers keep Google.Protobuf types out of the caller's signature. + public static class TrackPublishOptionsExtensions + { + public static TrackPublishOptions WithPacketTrailerFeatures( + this TrackPublishOptions options, + params PacketTrailerFeature[] features) + { + foreach (var feature in features) + options.PacketTrailerFeatures.Add(feature); + return options; + } + } } diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs index 5f62c671..1ef072d2 100644 --- a/Runtime/Scripts/RtcVideoSource.cs +++ b/Runtime/Scripts/RtcVideoSource.cs @@ -30,6 +30,12 @@ public enum VideoStreamSource /// Called when we receive a new texture (first texture or the resolution changed) public event TextureReceiveDelegate TextureReceived; + public delegate FrameMetadata FrameMetadataDelegate(); + /// Invoked once per outgoing frame. Return null (default) to send no trailer. + /// To actually serialize the trailer onto RTP, also enable the matching + /// PacketTrailerFeatures on the TrackPublishOptions used at publish time. + public FrameMetadataDelegate MetadataProvider { get; set; } + protected Texture2D _previewTexture; protected NativeArray _captureBuffer; protected VideoStreamSource _sourceType; @@ -175,6 +181,8 @@ protected virtual bool SendFrame() var now = DateTimeOffset.UtcNow; capture.TimestampUs = now.ToUnixTimeMilliseconds() * 1000 + (now.Ticks % TimeSpan.TicksPerMillisecond) / 10; capture.Buffer = buffer; + var metadata = MetadataProvider?.Invoke(); + if (metadata != null) capture.Metadata = metadata; using var response = request.Send(); _reading = false; _requestPending = false; diff --git a/Runtime/Scripts/VideoFrame.cs b/Runtime/Scripts/VideoFrame.cs index 7d263dd7..e2bf66d1 100644 --- a/Runtime/Scripts/VideoFrame.cs +++ b/Runtime/Scripts/VideoFrame.cs @@ -12,12 +12,14 @@ public sealed class VideoFrame public long Timestamp; public VideoRotation Rotation; + public FrameMetadata Metadata; - public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation) + public VideoFrame(VideoBufferInfo info, long timeStamp, VideoRotation rotation, FrameMetadata metadata = null) { _info = info; Timestamp = timeStamp; Rotation = rotation; + Metadata = metadata; } } diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 8d2fa5e9..0c1a148d 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -242,7 +242,7 @@ private void OnVideoStreamEvent(VideoStreamEvent e) // Avoid allocating VideoFrame objects when nobody is observing them. if (FrameReceived != null) { - var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation); + var frame = new VideoFrame(frameInfo, e.FrameReceived.TimestampUs, e.FrameReceived.Rotation, e.FrameReceived.Metadata); FrameReceived.Invoke(frame); } } diff --git a/Tests/PlayMode/TrackTests.cs b/Tests/PlayMode/TrackTests.cs index 88d5416d..62a723bb 100644 --- a/Tests/PlayMode/TrackTests.cs +++ b/Tests/PlayMode/TrackTests.cs @@ -149,7 +149,7 @@ public IEnumerator RemoteTrackPublication_SetVideoQuality_DoesNotThrow() var publisherRoom = context.Rooms[0]; var subscriberRoom = context.Rooms[1]; - var videoSource = new StubVideoSource(); + var videoSource = new TestVideoSource(); var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom); // Video track uses a stub source that never pushes frames. TrackSubscribed may not @@ -283,10 +283,10 @@ public IEnumerator RemoteTrackPublication_PublisherDisablesCamera_UpdatesFlagAnd var publisherRoom = context.Rooms[0]; var subscriberRoom = context.Rooms[1]; - var videoSource = new StubVideoSource(); + var videoSource = new TestVideoSource(); var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, videoSource, publisherRoom); - // StubVideoSource never pushes frames, so TrackSubscribed may not fire on the + // TestVideoSource (pushFrames=false) never pushes frames, so TrackSubscribed may not fire on the // subscriber. The RemoteTrackPublication still propagates via TrackPublished. var publishedExp = new Expectation(timeoutSeconds: 10f); subscriberRoom.TrackPublished += (_, _) => publishedExp.Fulfill(); diff --git a/Tests/PlayMode/Utils/CoroutineRunner.cs b/Tests/PlayMode/Utils/CoroutineRunner.cs new file mode 100644 index 00000000..b8f6db60 --- /dev/null +++ b/Tests/PlayMode/Utils/CoroutineRunner.cs @@ -0,0 +1,12 @@ +using UnityEngine; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Empty MonoBehaviour used by tests to host long-running coroutines + /// (e.g. and ) + /// that the test itself cannot host because [UnityTest] bodies must yield + /// back to the test runner. + /// + public class CoroutineRunner : MonoBehaviour { } +} diff --git a/Tests/PlayMode/Utils/StubVideoSource.cs.meta b/Tests/PlayMode/Utils/CoroutineRunner.cs.meta similarity index 60% rename from Tests/PlayMode/Utils/StubVideoSource.cs.meta rename to Tests/PlayMode/Utils/CoroutineRunner.cs.meta index 791a3108..76434df5 100644 --- a/Tests/PlayMode/Utils/StubVideoSource.cs.meta +++ b/Tests/PlayMode/Utils/CoroutineRunner.cs.meta @@ -1,11 +1,11 @@ fileFormatVersion: 2 -guid: 158c662279bb240c387fa142f26314df +guid: 85dedc359b4346d1bedd01817cedf75e MonoImporter: externalObjects: {} serializedVersion: 2 defaultReferences: [] executionOrder: 0 icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/Utils/StubVideoSource.cs b/Tests/PlayMode/Utils/StubVideoSource.cs deleted file mode 100644 index 865258e1..00000000 --- a/Tests/PlayMode/Utils/StubVideoSource.cs +++ /dev/null @@ -1,31 +0,0 @@ -using LiveKit.Proto; - -namespace LiveKit.PlayModeTests.Utils -{ - /// - /// Test-only that registers with FFI but never - /// pushes frames. Used for tests that only require the video publication - /// to propagate to subscribers via signaling (e.g. RemoteTrackPublication - /// APIs that operate on metadata) without actual media flow. - /// - public sealed class StubVideoSource : RtcVideoSource - { - private readonly int _width; - private readonly int _height; - - public override int GetWidth() => _width; - public override int GetHeight() => _height; - - protected override VideoRotation GetVideoRotation() => VideoRotation._0; - - protected override bool ReadBuffer() => false; - - public StubVideoSource(int width = 16, int height = 16) - : base(VideoStreamSource.Texture, VideoBufferType.Rgba) - { - _width = width; - _height = height; - Init(); - } - } -} diff --git a/Tests/PlayMode/Utils/TestVideoSource.cs b/Tests/PlayMode/Utils/TestVideoSource.cs new file mode 100644 index 00000000..2689b837 --- /dev/null +++ b/Tests/PlayMode/Utils/TestVideoSource.cs @@ -0,0 +1,53 @@ +using LiveKit.Proto; +using Unity.Collections; + +namespace LiveKit.PlayModeTests.Utils +{ + /// + /// Test-only registered with FFI at a fixed + /// resolution. Two modes via : + /// + /// false (default): never pushes frames. Use when the test only + /// needs the publication to propagate via signaling (e.g. + /// APIs that operate on metadata). + /// true: pushes a continuous stream of zero-filled RGBA frames. + /// Use when the test needs actual media flow (e.g. validating per-frame + /// metadata round-trips through the FFI / RTP path). + /// + /// + public sealed class TestVideoSource : RtcVideoSource + { + private readonly int _width; + private readonly int _height; + private readonly bool _pushFrames; + + public override int GetWidth() => _width; + public override int GetHeight() => _height; + + protected override VideoRotation GetVideoRotation() => VideoRotation._0; + + protected override bool ReadBuffer() + { + if (!_pushFrames) return false; + + if (!_captureBuffer.IsCreated) + { + _captureBuffer = new NativeArray( + _width * _height * 4, + Allocator.Persistent, + NativeArrayOptions.ClearMemory); + } + _requestPending = true; + return false; + } + + public TestVideoSource(bool pushFrames = false, int width = 16, int height = 16) + : base(VideoStreamSource.Texture, VideoBufferType.Rgba) + { + _width = width; + _height = height; + _pushFrames = pushFrames; + Init(); + } + } +} diff --git a/Tests/PlayMode/Utils/TestVideoSource.cs.meta b/Tests/PlayMode/Utils/TestVideoSource.cs.meta new file mode 100644 index 00000000..2b80cd2b --- /dev/null +++ b/Tests/PlayMode/Utils/TestVideoSource.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 7e4ca10e199b4df7ba2eb6de53497882 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Tests/PlayMode/VideoFrameMetadataTests.cs b/Tests/PlayMode/VideoFrameMetadataTests.cs new file mode 100644 index 00000000..c4c8108c --- /dev/null +++ b/Tests/PlayMode/VideoFrameMetadataTests.cs @@ -0,0 +1,103 @@ +using System.Collections; +using LiveKit.PlayModeTests.Utils; +using LiveKit.Proto; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.TestTools; + +namespace LiveKit.PlayModeTests +{ + public class VideoFrameMetadataTests + { + const string VideoTrackName = "metadata-video-track"; + + static (TestRoomContext.ConnectionOptions publisher, TestRoomContext.ConnectionOptions subscriber) TwoPeers() + { + var publisher = TestRoomContext.ConnectionOptions.Default; + publisher.Identity = "metadata-publisher"; + var subscriber = TestRoomContext.ConnectionOptions.Default; + subscriber.Identity = "metadata-subscriber"; + return (publisher, subscriber); + } + + [UnityTest, Category("E2E")] + public IEnumerator VideoFrame_AttachedMetadata_ReceivedOnSubscriber() + { + var (publisher, subscriber) = TwoPeers(); + using var context = new TestRoomContext(new[] { publisher, subscriber }); + yield return context.ConnectAll(); + Assert.IsNull(context.ConnectionError, context.ConnectionError); + + var publisherRoom = context.Rooms[0]; + var subscriberRoom = context.Rooms[1]; + + const uint expectedFrameId = 42u; + const ulong expectedUserTs = 0x1122334455667788UL; + + var source = new TestVideoSource(pushFrames: true); + source.MetadataProvider = () => new FrameMetadata + { + FrameId = expectedFrameId, + UserTimestamp = expectedUserTs, + }; + + var localTrack = LocalVideoTrack.CreateVideoTrack(VideoTrackName, source, publisherRoom); + + var subscribedExp = new Expectation(timeoutSeconds: 10f); + RemoteVideoTrack receivedRemoteTrack = null; + subscriberRoom.TrackSubscribed += (track, _, _) => + { + if (track is RemoteVideoTrack rv) + { + receivedRemoteTrack = rv; + subscribedExp.Fulfill(); + } + }; + + var options = new TrackPublishOptions + { + Source = TrackSource.SourceCamera, + }.WithPacketTrailerFeatures( + PacketTrailerFeature.PtfUserTimestamp, + PacketTrailerFeature.PtfFrameId); + var pub = publisherRoom.LocalParticipant.PublishTrack(localTrack, options); + yield return pub; + Assert.IsFalse(pub.IsError); + + // Host the source's Update coroutine on a throwaway MonoBehaviour so the + // test body can yield on Expectations without owning the producer loop. + var runnerObj = new GameObject("metadata-test-runner"); + var runner = runnerObj.AddComponent(); + source.Start(); + runner.StartCoroutine(source.Update()); + + yield return subscribedExp.Wait(); + Assert.IsNull(subscribedExp.Error); + Assert.IsNotNull(receivedRemoteTrack); + + var stream = new VideoStream(receivedRemoteTrack); + var metadataExp = new Expectation(timeoutSeconds: 10f); + FrameMetadata receivedMetadata = null; + stream.FrameReceived += frame => + { + if (receivedMetadata != null || frame.Metadata == null) return; + receivedMetadata = frame.Metadata; + metadataExp.Fulfill(); + }; + stream.Start(); + runner.StartCoroutine(stream.Update()); + + yield return metadataExp.Wait(); + Assert.IsNull(metadataExp.Error, "expected a frame with metadata within timeout"); + Assert.IsNotNull(receivedMetadata); + Assert.AreEqual(expectedFrameId, receivedMetadata.FrameId, "frame_id mismatch"); + Assert.AreEqual(expectedUserTs, receivedMetadata.UserTimestamp, "user_timestamp mismatch"); + + source.Stop(); + stream.Stop(); + stream.Dispose(); + source.Dispose(); + Object.Destroy(runnerObj); + } + } +} diff --git a/Tests/PlayMode/VideoFrameMetadataTests.cs.meta b/Tests/PlayMode/VideoFrameMetadataTests.cs.meta new file mode 100644 index 00000000..35011034 --- /dev/null +++ b/Tests/PlayMode/VideoFrameMetadataTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b2926ee0c31a473294e3ecdcddd0a227 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: