From 8f0a59f575ba19a9bc09502688a080fdc6158945 Mon Sep 17 00:00:00 2001 From: Mario Di Vece Date: Sun, 23 Sep 2018 22:12:44 -0500 Subject: [PATCH] Seek Improvements and Indexing (#280) * starting with seek indexing. Related to issue #139 * continued work on seek indexing. * Generalizing sorted binary search algorithm. Continued work on seek indices. See issues #250 and #139 * improved and simplified seek logic. * minor renaming and added seek error checking. * logging when using a video seek index entry. * fixing frame start offset by offseting them by main component. --- .../Commands/CommandManager.cs | 12 +- Unosquare.FFME.Common/Commands/SeekCommand.cs | 72 +++-- Unosquare.FFME.Common/Decoding/AudioFrame.cs | 5 +- .../Decoding/MediaContainer.cs | 282 ++++++------------ Unosquare.FFME.Common/Decoding/MediaFrame.cs | 17 +- .../Decoding/SeekRequirement.cs | 30 -- .../Decoding/SubtitleFrame.cs | 3 +- .../Decoding/VideoComponent.cs | 11 + Unosquare.FFME.Common/Decoding/VideoFrame.cs | 12 +- Unosquare.FFME.Common/MediaEngine.Static.cs | 84 ++++-- Unosquare.FFME.Common/MediaEngine.Workers.cs | 31 ++ .../Primitives/MediaBlockBuffer.cs | 50 +--- Unosquare.FFME.Common/Shared/Extensions.cs | 48 ++- Unosquare.FFME.Common/Shared/MediaBlock.cs | 16 +- Unosquare.FFME.Common/Shared/MediaOptions.cs | 7 + .../Shared/VideoSeekIndex.cs | 253 ++++++++++++++++ .../Shared/VideoSeekIndexEntry.cs | 117 ++++++++ .../MainWindow.MediaEvents.cs | 88 +++++- .../Properties/AssemblyInfo.cs | 2 +- .../ViewModels/PlaylistViewModel.cs | 10 + .../Properties/AssemblyInfo.cs | 2 +- 21 files changed, 792 insertions(+), 360 deletions(-) delete mode 100644 Unosquare.FFME.Common/Decoding/SeekRequirement.cs create mode 100644 Unosquare.FFME.Common/Shared/VideoSeekIndex.cs create mode 100644 Unosquare.FFME.Common/Shared/VideoSeekIndexEntry.cs diff --git a/Unosquare.FFME.Common/Commands/CommandManager.cs b/Unosquare.FFME.Common/Commands/CommandManager.cs index 9b2cde9be..ec091dbf7 100644 --- a/Unosquare.FFME.Common/Commands/CommandManager.cs +++ b/Unosquare.FFME.Common/Commands/CommandManager.cs @@ -38,7 +38,7 @@ internal sealed class CommandManager : IDisposable, ILoggingSource private bool m_IsDisposed; private DirectCommandBase CurrentDirectCommand; - private CommandBase CurrentQueueCommand; + private CommandBase ExecutingQueueCommand; #endregion @@ -302,7 +302,7 @@ public CommandType ExecuteNextQueuedCommand() { command = CommandQueue[0]; CommandQueue.RemoveAt(0); - CurrentQueueCommand = command; + ExecutingQueueCommand = command; } } @@ -339,7 +339,7 @@ public CommandType ExecuteNextQueuedCommand() DecrementPendingSeeks(); } - CurrentQueueCommand = null; + ExecutingQueueCommand = null; } } @@ -497,10 +497,10 @@ private async Task ExecutePriorityCommand(CommandType commandType) CommandBase currentCommand = null; lock (QueueLock) { - if (CurrentQueueCommand != null && - CurrentQueueCommand.CommandType == commandType) + if (ExecutingQueueCommand != null && + ExecutingQueueCommand.CommandType == commandType) { - currentCommand = CurrentQueueCommand; + currentCommand = ExecutingQueueCommand; } if (currentCommand == null) diff --git a/Unosquare.FFME.Common/Commands/SeekCommand.cs b/Unosquare.FFME.Common/Commands/SeekCommand.cs index e48e5799b..6e5f12d41 100644 --- a/Unosquare.FFME.Common/Commands/SeekCommand.cs +++ b/Unosquare.FFME.Common/Commands/SeekCommand.cs @@ -92,18 +92,18 @@ protected override void PerformActions() // Mark for debugger output hasDecoderSeeked = true; - // Signal the starting state clearing the packet buffer cache - m.Container.Components.ClearQueuedPackets(flushBuffers: true); - // wait for the current reading and decoding cycles // to finish. We don't want to interfere with reading in progress // or decoding in progress. For decoding we already know we are not // in a cycle because the decoding worker called this logic. m.PacketReadingCycle.Wait(); + // Signal the starting state clearing the packet buffer cache + m.Container.Components.ClearQueuedPackets(flushBuffers: true); + // Capture seek target adjustment var adjustedSeekTarget = TargetPosition; - if (mainBlocks.IsMonotonic) + if (TargetPosition != TimeSpan.Zero && mainBlocks.IsMonotonic) { var targetSkewTicks = Convert.ToInt64( mainBlocks.MonotonicDuration.Ticks * (mainBlocks.Capacity / 2d)); @@ -112,38 +112,48 @@ protected override void PerformActions() adjustedSeekTarget = TimeSpan.FromTicks(adjustedSeekTarget.Ticks - targetSkewTicks); } - // Clear Blocks and frames, reset the render times - foreach (var mt in all) - { - m.Blocks[mt].Clear(); - m.InvalidateRenderer(mt); - } - // Populate frame queues with after-seek operation - var frames = m.Container.Seek(adjustedSeekTarget); - m.State.UpdateMediaEnded(false, TimeSpan.Zero); - - // Clear all the blocks. We don't need them - foreach (var kvp in m.Blocks) - kvp.Value.Clear(); - - // Create the blocks from the obtained seek frames - foreach (var frame in frames) - m.Blocks[frame.MediaType]?.Add(frame, m.Container); - - // Now read blocks until we have reached at least the Target Position - // TODO: This might not be entirely right - while (m.ShouldReadMorePackets - && mainBlocks.IsFull == false - && mainBlocks.IsInRange(TargetPosition) == false) + var firstFrame = m.Container.Seek(adjustedSeekTarget); + if (firstFrame != null) { - // Read the next packet - m.Container.Read(); + // Ensure we signal media has not ended + m.State.UpdateMediaEnded(false, TimeSpan.Zero); + // Clear Blocks and frames, reset the render times foreach (var mt in all) { - if (m.Blocks[mt].IsFull == false) - m.Blocks[mt].Add(m.Container.Components[mt].ReceiveNextFrame(), m.Container); + m.Blocks[mt].Clear(); + m.InvalidateRenderer(mt); + } + + // Create the blocks from the obtained seek frames + m.Blocks[firstFrame.MediaType]?.Add(firstFrame, m.Container); + + // Decode all available queued packets into the media component blocks + foreach (var mt in all) + { + while (m.Blocks[mt].IsFull == false) + { + var frame = m.Container.Components[mt].ReceiveNextFrame(); + if (frame == null) break; + m.Blocks[mt].Add(frame, m.Container); + } + } + + // Align to the exact requested position on the main component + while (m.ShouldReadMorePackets) + { + // Check if we are already in range + if (mainBlocks.IsInRange(TargetPosition)) break; + + // Read the next packet + var packetType = m.Container.Read(); + var blocks = m.Blocks[packetType]; + if (blocks == null) continue; + + // Get the next frame + if (blocks.RangeEndTime.Ticks < TargetPosition.Ticks || blocks.IsFull == false) + blocks.Add(m.Container.Components[packetType].ReceiveNextFrame(), m.Container); } } diff --git a/Unosquare.FFME.Common/Decoding/AudioFrame.cs b/Unosquare.FFME.Common/Decoding/AudioFrame.cs index 0d6cc5996..e28ba57d8 100644 --- a/Unosquare.FFME.Common/Decoding/AudioFrame.cs +++ b/Unosquare.FFME.Common/Decoding/AudioFrame.cs @@ -30,14 +30,15 @@ internal AudioFrame(AVFrame* frame, MediaComponent component) : base(frame, component, MediaType.Audio) { // Compute the start time. + var mainOffset = component.Container.Components.Main.StartTime; frame->pts = frame->best_effort_timestamp; HasValidStartTime = frame->pts != ffmpeg.AV_NOPTS_VALUE; StartTime = frame->pts == ffmpeg.AV_NOPTS_VALUE ? TimeSpan.FromTicks(0) : - TimeSpan.FromTicks(frame->pts.ToTimeSpan(StreamTimeBase).Ticks - component.Container.MediaStartTime.Ticks); + TimeSpan.FromTicks(frame->pts.ToTimeSpan(StreamTimeBase).Ticks - mainOffset.Ticks); // Compute the audio frame duration - Duration = frame->pkt_duration != 0 ? + Duration = frame->pkt_duration > 0 ? frame->pkt_duration.ToTimeSpan(StreamTimeBase) : TimeSpan.FromTicks(Convert.ToInt64(TimeSpan.TicksPerMillisecond * 1000d * frame->nb_samples / frame->sample_rate)); diff --git a/Unosquare.FFME.Common/Decoding/MediaContainer.cs b/Unosquare.FFME.Common/Decoding/MediaContainer.cs index a92953dc1..08caef749 100644 --- a/Unosquare.FFME.Common/Decoding/MediaContainer.cs +++ b/Unosquare.FFME.Common/Decoding/MediaContainer.cs @@ -270,7 +270,7 @@ public long MediaStreamSize /// /// Gets a value indicating whether the underlying media is seekable. /// - public bool IsStreamSeekable => MediaDuration.TotalSeconds > 0 && MediaDuration != TimeSpan.MinValue; + public bool IsStreamSeekable => MediaDuration.Ticks > 0; /// /// Gets a value indicating whether this container represents live media. @@ -368,10 +368,9 @@ public void Open() } /// - /// Seeks to the specified position in the stream. This method attempts to do so as - /// precisely as possible, returning decoded frames of all available media type components - /// just before or right on the requested position. The position must be given in 0-based time, - /// so it converts component stream start time offset to absolute, 0-based time. + /// Seeks to the specified position in the main stream component. + /// Returns the keyframe on or before the specified position. Most of the times + /// you will need to keep reading packets and receiving frames to reach the exact position. /// Pass TimeSpan.Zero to seek to the beginning of the stream. /// /// The position. @@ -379,7 +378,7 @@ public void Open() /// The list of media frames /// /// No input context initialized - public List Seek(TimeSpan position) + public MediaFrame Seek(TimeSpan position) { lock (ReadSyncRoot) { @@ -922,11 +921,6 @@ private MediaType[] StreamCreateComponents() // Picture attachments are only required after the first read or after a seek. StateRequiresPictureAttachments = true; - // Update start time and duration based on main component - MediaDuration = Components.Main.Duration; - MediaStartTime = Components.Main.StartTime == TimeSpan.MinValue ? - TimeSpan.Zero : Components.Main.StartTime; - // Output start time offsets. this.LogInfo(Aspects.Container, $"Timing Offsets - Main Component: {Components.MainMediaType}; " + @@ -1045,32 +1039,38 @@ private int OnStreamReadInterrupt(void* opaque) } /// - /// Seeks to the exact or prior frame of the main stream. - /// Supports byte seeking. Target time is in absolute, zero-based time. + /// Seeks to the closest and lesser or equal key frame on the main component + /// Target time is in absolute, zero-based time. /// /// The target time in absolute, 0-based time. /// /// The list of media frames /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private List StreamSeek(TimeSpan targetTimeAbsolute) + private MediaFrame StreamSeek(TimeSpan targetTimeAbsolute) { - // Create the output result object - var result = new List(256); - var seekRequirement = Components.MainMediaType == MediaType.Audio ? - SeekRequirement.MainComponentOnly : SeekRequirement.AudioAndVideo; - #region Setup - // Capture absolute, 0-based target time and clamp it - var targetTime = TimeSpan.FromTicks(targetTimeAbsolute.Ticks); + // Select the main component + var main = Components.Main; + if (main == null) return null; + MediaFrame frame = null; + + // Stream seeking by main component + // The backward flag means that we want to seek to at MOST the target position + var seekFlags = ffmpeg.AVSEEK_FLAG_BACKWARD; + var streamIndex = main.StreamIndex; + var timeBase = main.Stream->time_base; + + // Compute the absolute maximum 0-based target time which is simply the duration. + var maxTargetTimeTicks = main.Duration.Ticks > 0 ? main.Duration.Ticks : 0; + + // Adjust 0-based target time and clamp it + var targetPosition = TimeSpan.FromTicks(targetTimeAbsolute.Ticks.Clamp(0L, maxTargetTimeTicks)); // A special kind of seek is the zero seek. Execute it if requested. - if (targetTime <= TimeSpan.Zero) - { - StreamSeekToStart(result, seekRequirement); - return result; // this may or may not have the start frames - } + if (targetPosition.Ticks <= 0) + return StreamSeekToStart(); // Cancel the seek operation if the stream does not support it. if (IsStreamSeekable == false) @@ -1078,27 +1078,9 @@ private List StreamSeek(TimeSpan targetTimeAbsolute) this.LogWarning(Aspects.EngineCommand, "Unable to seek. Underlying stream does not support seeking."); - return result; + return null; } - // Select the main component - var main = Components.Main; - if (main == null) return result; - - // clamp the minimum time to zero (can't be less than 0) - if (targetTime.Ticks + main.StartTime.Ticks < main.StartTime.Ticks) - targetTime = TimeSpan.Zero; - - // Clamp the maximum seek value to main component's duration (can't be more than duration) - if (targetTime.Ticks > main.Duration.Ticks) - targetTime = TimeSpan.FromTicks(main.Duration.Ticks); - - // Stream seeking by main component - // The backward flag means that we want to seek to at MOST the target position - var seekFlags = ffmpeg.AVSEEK_FLAG_BACKWARD; - var streamIndex = main.StreamIndex; - var timeBase = main.Stream->time_base; - // Perform the stream seek int seekResult; var startPos = StreamPosition; @@ -1111,7 +1093,23 @@ private List StreamSeek(TimeSpan targetTimeAbsolute) // if the seeking is not successful we decrement this time and try the seek // again by subtracting 1 second from it. var startTime = DateTime.UtcNow; - var streamSeekRelativeTime = TimeSpan.FromTicks(targetTime.Ticks + main.StartTime.Ticks); // Offset by start time + var mainOffset = main.StartTime; + var streamSeekRelativeTime = TimeSpan.FromTicks(targetPosition.Ticks + mainOffset.Ticks); // Offset by start time + var indexTimestamp = ffmpeg.AV_NOPTS_VALUE; + + // Help the initial position seek time. + if (main is VideoComponent videoComponent && videoComponent.SeekIndex.Count > 0) + { + var entryIndex = videoComponent.SeekIndex.StartIndexOf(targetPosition); + if (entryIndex >= 0) + { + var entry = videoComponent.SeekIndex[entryIndex]; + this.LogDebug(Aspects.Container, + $"SEEK IX: Seek index entry {entryIndex} found. " + + $"Entry Position: {entry.StartTime.Format()} | Target: {targetPosition.Format()}"); + indexTimestamp = entry.PresentationTime; + } + } // Perform long seeks until we end up with a relative target time where decoding // of frames before or on target time is possible. @@ -1119,7 +1117,14 @@ private List StreamSeek(TimeSpan targetTimeAbsolute) while (isAtStartOfStream == false) { // Compute the seek target, mostly based on the relative Target Time - var seekTarget = streamSeekRelativeTime.ToLong(timeBase); + var seekTimestamp = streamSeekRelativeTime.ToLong(timeBase); + + // If we have an index timestamp, then use it. + if (indexTimestamp != ffmpeg.AV_NOPTS_VALUE) + { + seekTimestamp = indexTimestamp; + indexTimestamp = ffmpeg.AV_NOPTS_VALUE; + } // Perform the seek. There is also avformat_seek_file which is the older version of av_seek_frame // Check if we are seeking before the start of the stream in this cycle. If so, simply seek to the @@ -1131,17 +1136,16 @@ private List StreamSeek(TimeSpan targetTimeAbsolute) else { StreamReadInterruptStartTime.Value = DateTime.UtcNow; - if (streamSeekRelativeTime.Ticks <= main.StartTime.Ticks) + if (streamSeekRelativeTime.Ticks <= mainOffset.Ticks) { - seekTarget = main.StartTime.ToLong(main.Stream->time_base); - streamIndex = main.StreamIndex; + seekTimestamp = mainOffset.ToLong(main.Stream->time_base); isAtStartOfStream = true; } - seekResult = ffmpeg.av_seek_frame(InputContext, streamIndex, seekTarget, seekFlags); + seekResult = ffmpeg.av_seek_frame(InputContext, streamIndex, seekTimestamp, seekFlags); this.LogTrace(Aspects.Container, $"SEEK L: Elapsed: {startTime.FormatElapsed()} | Target: {streamSeekRelativeTime.Format()} " + - $"| Seek: {seekTarget.Format()} | P0: {startPos.Format(1024)} | P1: {StreamPosition.Format(1024)} "); + $"| Seek: {seekTimestamp.Format()} | P0: {startPos.Format(1024)} | P1: {StreamPosition.Format(1024)} "); } // Flush the buffered packets and codec on every seek. @@ -1158,51 +1162,40 @@ private List StreamSeek(TimeSpan targetTimeAbsolute) break; } - // Read and decode frames for all components and check if the decoded frames - // are on or right before the target time. - StreamSeekDecode(result, targetTime, seekRequirement); - - var firstAudioFrame = result.FirstOrDefault(f => f.MediaType == MediaType.Audio && f.StartTime <= targetTime); - var firstVideoFrame = result.FirstOrDefault(f => f.MediaType == MediaType.Video && f.StartTime <= targetTime); - - var isAudioSeekInRange = Components.HasAudio == false - || (firstAudioFrame == null && Components.MainMediaType != MediaType.Audio) - || (firstAudioFrame != null && firstAudioFrame.StartTime <= targetTime); + // Get the main component position + frame = StreamPositionDecode(main); - var isVideoSeekInRange = Components.HasVideo == false - || (firstVideoFrame == null && Components.MainMediaType != MediaType.Video) - || (firstVideoFrame != null && firstVideoFrame.StartTime <= targetTime); - - // If we have the correct range, no further processing is required. - if (isAudioSeekInRange && isVideoSeekInRange) - break; - - // At this point the result is useless. Simply discard the decoded frames - foreach (var frame in result) frame.Dispose(); - result.Clear(); + // If we could not read a frame from the main component or + // if the first decoded frame is past the target time + // try again with a lower relative time. + if (frame == null || frame.StartTime.Ticks > targetPosition.Ticks) + { + streamSeekRelativeTime = streamSeekRelativeTime.Subtract(TimeSpan.FromSeconds(1)); + frame?.Dispose(); + frame = null; + continue; + } - // Subtract 1 second from the relative target time. - // a new seek target will be computed and we will do a av_seek_frame again. - streamSeekRelativeTime = streamSeekRelativeTime.Subtract(TimeSpan.FromSeconds(1)); + // At this point frame contains the + // prior keyframe to the seek target + break; } this.LogTrace(Aspects.Container, $"SEEK R: Elapsed: {startTime.FormatElapsed()} | Target: {streamSeekRelativeTime.Format()} " + $"| Seek: {default(long).Format()} | P0: {startPos.Format(1024)} | P1: {StreamPosition.Format(1024)} "); - return result; - #endregion + return frame; + #endregion } /// /// Seeks to the position at the start of the stream. /// - /// The result. - /// The seek requirement. - /// The number of read and decode cycles + /// The first frame of the main component. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int StreamSeekToStart(List result, SeekRequirement seekRequirement) + private MediaFrame StreamSeekToStart() { var main = Components.Main; var seekTarget = main.StartTime.ToLong(main.Stream->time_base); @@ -1211,7 +1204,7 @@ private int StreamSeekToStart(List result, SeekRequirement seekRequi StreamReadInterruptStartTime.Value = DateTime.UtcNow; - // TODO: seekTarget might need further adjustment. Maybe seek to long.MinValue? + // Execute the seek to start of main component var seekResult = ffmpeg.av_seek_frame(InputContext, streamIndex, seekTarget, seekFlags); // Flush packets, state, and codec buffers @@ -1219,128 +1212,33 @@ private int StreamSeekToStart(List result, SeekRequirement seekRequi StateRequiresPictureAttachments = true; IsAtEndOfStream = false; - if (seekResult >= 0) return StreamSeekDecode(result, TimeSpan.Zero, seekRequirement); + if (seekResult >= 0) + return StreamPositionDecode(main); this.LogWarning(Aspects.EngineCommand, $"SEEK 0: {nameof(StreamSeekToStart)} operation failed. Error code {seekResult}: {FFInterop.DecodeMessage(seekResult)}"); - return 0; - } - - /// - /// Reads and decodes packets until the required media components have frames on or right before the target time. - /// - /// The list of frames that is currently being processed. Frames will be added here. - /// The target time in absolute 0-based time. - /// The requirement. - /// The number of decoded frames - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int StreamSeekDecode(List result, TimeSpan targetTime, SeekRequirement requirement) - { - var readSeekCycles = 0; - MediaFrame frame; - - // Create a holder of frame lists; one for each type of media - var outputFrames = new Dictionary>(); - foreach (var mediaType in Components.MediaTypes) - outputFrames[mediaType] = new List(128); - - // Create a component requirement - var requiredComponents = new List(8); - if (requirement == SeekRequirement.AllComponents) - { - requiredComponents.AddRange(Components.MediaTypes.ToArray()); - } - else if (requirement == SeekRequirement.AudioAndVideo) - { - if (Components.HasVideo) requiredComponents.Add(MediaType.Video); - if (Components.HasAudio) requiredComponents.Add(MediaType.Audio); - } - else - { - requiredComponents.Add(Components.MainMediaType); - } - - // Start reading and decoding util we reach the target - var isDoneSeeking = false; - while (SignalAbortReadsRequested.Value == false && IsAtEndOfStream == false && isDoneSeeking == false) - { - readSeekCycles++; - - // Read the next packet - var mediaType = Read(); - - // Check if packet contains valid output - if (outputFrames.ContainsKey(mediaType) == false) - continue; - - // Decode and add the frames to the corresponding output - frame = Components[mediaType].ReceiveNextFrame(); - if (frame != null) outputFrames[mediaType].Add(frame); - - // keep the frames list short - foreach (var componentFrames in outputFrames.Values) - { - // cleanup frames if the output becomes too big - if (componentFrames.Count >= 24) - StreamSeekDiscardFrames(componentFrames, targetTime); - } - - // Based on the required components check the target time ranges - isDoneSeeking = outputFrames.Where(t => requiredComponents.Contains(t.Key)).Select(t => t.Value) - .All(frames => frames.Count > 0 && frames.Max(f => f.StartTime) >= targetTime); - } - - // Perform one final cleanup and aggregate the frames into a single, - // interleaved collection - foreach (var kvp in outputFrames) - { - var componentFrames = kvp.Value; - StreamSeekDiscardFrames(componentFrames, targetTime); - result.AddRange(componentFrames); - } - - result.Sort(); - return readSeekCycles; + return null; } /// - /// Drops the seek frames that are no longer needed. - /// Target time should be provided in absolute, 0-based time + /// Reads from the stream and receives the next available frame + /// from the specified component at the current stream position. + /// This is a helper method for seeking logic. /// - /// The frames. - /// The target time. - /// The number of dropped frames + /// The component. + /// The next available frame [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int StreamSeekDiscardFrames(List frames, TimeSpan targetTime) + private MediaFrame StreamPositionDecode(MediaComponent component) { - var result = 0; - if (frames.Count <= 1) return result; - frames.Sort(); - - var framesToDrop = new List(frames.Count); - - for (var i = 0; i < frames.Count - 1; i++) - { - var currentFrame = frames[i]; - var nextFrame = frames[i + 1]; - - if (currentFrame.StartTime >= targetTime) - break; - if (currentFrame.StartTime < targetTime && nextFrame.StartTime <= targetTime) - framesToDrop.Add(i); - } - - for (var i = framesToDrop.Count - 1; i >= 0; i--) + while (SignalAbortReadsRequested.Value == false && IsAtEndOfStream == false) { - var dropIndex = framesToDrop[i]; - var frame = frames[dropIndex]; - frames.RemoveAt(dropIndex); - frame.Dispose(); - result++; + var frame = component.ReceiveNextFrame(); + if (frame != null) return frame; + Read(); } - return result; + return null; } #endregion diff --git a/Unosquare.FFME.Common/Decoding/MediaFrame.cs b/Unosquare.FFME.Common/Decoding/MediaFrame.cs index 00c0ea097..379326ced 100644 --- a/Unosquare.FFME.Common/Decoding/MediaFrame.cs +++ b/Unosquare.FFME.Common/Decoding/MediaFrame.cs @@ -27,7 +27,8 @@ protected MediaFrame(AVFrame* pointer, MediaComponent component, MediaType media { var packetSize = pointer->pkt_size; CompressedSize = packetSize > 0 ? packetSize : 0; - PresentationTimestamp = pointer->pts.ToTimeSpan(StreamTimeBase); + PresentationTime = pointer->pts; + DecodingTime = pointer->pkt_dts; } /// @@ -40,7 +41,8 @@ protected MediaFrame(AVSubtitle* pointer, MediaComponent component) { // TODO: Compressed size is simply an estimate CompressedSize = (int)pointer->num_rects * 256; - PresentationTimestamp = Convert.ToInt64(pointer->start_display_time).ToTimeSpan(StreamTimeBase); + PresentationTime = Convert.ToInt64(pointer->start_display_time); + DecodingTime = pointer->pts; } /// @@ -75,9 +77,16 @@ private MediaFrame(void* pointer, MediaComponent component, MediaType mediaType) public int CompressedSize { get; } /// - /// Gets the unadjusted, original presentation timestamp of the frame. + /// Gets the unadjusted, original presentation timestamp (PTS) of the frame. + /// This is in units. /// - public TimeSpan PresentationTimestamp { get; } + public long PresentationTime { get; } + + /// + /// Gets the unadjusted, original presentation timestamp (PTS) of the packet. + /// This is in units. + /// + public long DecodingTime { get; } /// /// Gets the start time of the frame. diff --git a/Unosquare.FFME.Common/Decoding/SeekRequirement.cs b/Unosquare.FFME.Common/Decoding/SeekRequirement.cs deleted file mode 100644 index 00091e23b..000000000 --- a/Unosquare.FFME.Common/Decoding/SeekRequirement.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace Unosquare.FFME.Decoding -{ - /// - /// Enumerates the seek target requirement levels. - /// - internal enum SeekRequirement - { - /// - /// Seek requirement is satisfied when - /// the main component has frames in the seek range. - /// This is the fastest option. - /// - MainComponentOnly, - - /// - /// Seek requirement is satisfied when - /// the both audio and video comps have frames in the seek range. - /// This is the recommended option. - /// - AudioAndVideo, - - /// - /// Seek requirement is satisfied when - /// ALL components have frames in the seek range - /// This is NOT recommended as it forces large amounts of - /// frames to get decoded in subtitle files. - /// - AllComponents - } -} diff --git a/Unosquare.FFME.Common/Decoding/SubtitleFrame.cs b/Unosquare.FFME.Common/Decoding/SubtitleFrame.cs index c4c42e00d..e9d472537 100644 --- a/Unosquare.FFME.Common/Decoding/SubtitleFrame.cs +++ b/Unosquare.FFME.Common/Decoding/SubtitleFrame.cs @@ -33,7 +33,8 @@ internal SubtitleFrame(AVSubtitle* frame, MediaComponent component) { // Extract timing information (pts for Subtitles is always in AV_TIME_BASE units) HasValidStartTime = frame->pts != ffmpeg.AV_NOPTS_VALUE; - var timeOffset = TimeSpan.FromTicks(frame->pts.ToTimeSpan(ffmpeg.AV_TIME_BASE).Ticks - component.Container.MediaStartTime.Ticks); + var mainOffset = component.Container.Components.Main.StartTime; + var timeOffset = TimeSpan.FromTicks(frame->pts.ToTimeSpan(ffmpeg.AV_TIME_BASE).Ticks - mainOffset.Ticks); // start_display_time and end_display_time are relative to timeOffset StartTime = TimeSpan.FromTicks(timeOffset.Ticks + Convert.ToInt64(frame->start_display_time).ToTimeSpan(StreamTimeBase).Ticks); diff --git a/Unosquare.FFME.Common/Decoding/VideoComponent.cs b/Unosquare.FFME.Common/Decoding/VideoComponent.cs index 405cbe7d6..6e160f75d 100644 --- a/Unosquare.FFME.Common/Decoding/VideoComponent.cs +++ b/Unosquare.FFME.Common/Decoding/VideoComponent.cs @@ -72,6 +72,11 @@ internal VideoComponent(MediaContainer container, int streamIndex) var aspectRatio = ffmpeg.av_d2q((double)FrameWidth / FrameHeight, int.MaxValue); DisplayAspectWidth = aspectRatio.num; DisplayAspectHeight = aspectRatio.den; + + var seekIndex = container.MediaOptions.VideoSeekIndex; + SeekIndex = seekIndex != null && seekIndex.StreamIndex == StreamIndex ? + new ReadOnlyCollection(seekIndex.Entries) : + new ReadOnlyCollection(new List(0)); } #endregion @@ -133,6 +138,12 @@ internal VideoComponent(MediaContainer container, int streamIndex) /// public bool IsUsingHardwareDecoding { get; private set; } + /// + /// Gets the video seek index for this component. + /// Returns null if it was not set in the media options. + /// + public ReadOnlyCollection SeekIndex { get; } + #endregion #region Methods diff --git a/Unosquare.FFME.Common/Decoding/VideoFrame.cs b/Unosquare.FFME.Common/Decoding/VideoFrame.cs index 276fca30a..23d8da1d8 100644 --- a/Unosquare.FFME.Common/Decoding/VideoFrame.cs +++ b/Unosquare.FFME.Common/Decoding/VideoFrame.cs @@ -30,6 +30,7 @@ internal VideoFrame(AVFrame* frame, VideoComponent component) : base(frame, component, MediaType.Video) { var timeBase = ffmpeg.av_guess_frame_rate(component.Container.InputContext, component.Stream, frame); + var mainOffset = component.Container.Components.Main.StartTime; var repeatFactor = 1d + (0.5d * frame->repeat_pict); Duration = frame->pkt_duration <= 0 ? @@ -39,15 +40,15 @@ internal VideoFrame(AVFrame* frame, VideoComponent component) // for video frames, we always get the best effort timestamp as dts and pts might // contain different times. frame->pts = frame->best_effort_timestamp; - HasValidStartTime = frame->pts != ffmpeg.AV_NOPTS_VALUE; StartTime = frame->pts == ffmpeg.AV_NOPTS_VALUE ? TimeSpan.FromTicks(0) : - TimeSpan.FromTicks(frame->pts.ToTimeSpan(StreamTimeBase).Ticks - component.Container.MediaStartTime.Ticks); + TimeSpan.FromTicks(frame->pts.ToTimeSpan(StreamTimeBase).Ticks - mainOffset.Ticks); EndTime = TimeSpan.FromTicks(StartTime.Ticks + Duration.Ticks); - // Picture Number and SMTPE TimeCode + // Picture Type, Number and SMTPE TimeCode + PictureType = frame->pict_type; DisplayPictureNumber = frame->display_picture_number == 0 ? Extensions.ComputePictureNumber(StartTime, Duration, 1) : frame->display_picture_number; @@ -95,6 +96,11 @@ internal VideoFrame(AVFrame* frame, VideoComponent component) /// public long DisplayPictureNumber { get; } + /// + /// Gets the video picture type. I frames are key frames. + /// + public AVPictureType PictureType { get; } + /// /// Gets the coded picture number set by the decoder. /// diff --git a/Unosquare.FFME.Common/MediaEngine.Static.cs b/Unosquare.FFME.Common/MediaEngine.Static.cs index b8c137288..cdc73114c 100644 --- a/Unosquare.FFME.Common/MediaEngine.Static.cs +++ b/Unosquare.FFME.Common/MediaEngine.Static.cs @@ -9,7 +9,6 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; - using System.Runtime.CompilerServices; public partial class MediaEngine { @@ -310,6 +309,58 @@ public static MediaInfo RetrieveMediaInfo(string sourceUrl) return container.MediaInfo; } + /// + /// Creates a viedo seek index. + /// + /// The source URL. + /// Index of the stream. Use -1 for automatic stream selection. + /// + /// The seek index object + /// + public static VideoSeekIndex CreateVideoSeekIndex(string sourceUrl, int streamIndex) + { + var result = new VideoSeekIndex(sourceUrl, -1); + + using (var container = new MediaContainer(sourceUrl, null, null)) + { + container.MediaOptions.IsAudioDisabled = true; + container.MediaOptions.IsVideoDisabled = false; + container.MediaOptions.IsSubtitleDisabled = true; + + if (streamIndex >= 0) + container.MediaOptions.VideoStream = container.MediaInfo.Streams[streamIndex]; + + container.Open(); + result.StreamIndex = container.Components.Video.StreamIndex; + while (container.IsStreamSeekable) + { + container.Read(); + var frames = container.Decode(); + foreach (var frame in frames) + { + try + { + if (frame.MediaType != MediaType.Video) + continue; + + // Check if the frame is a key frame and add it to the index. + result.TryAdd(frame as VideoFrame); + } + finally + { + frame.Dispose(); + } + } + + // We have reached the end of the stream. + if (frames.Count <= 0 && container.IsAtEndOfStream) + break; + } + } + + return result; + } + /// /// Reads all the blocks of the specified media type from the source url. /// @@ -361,37 +412,6 @@ internal static MediaBlockBuffer LoadBlocks(string sourceUrl, MediaType sourceTy } } - /// - /// Logs a block rendering operation as a Trace Message - /// if the debugger is attached. - /// - /// The block. - /// The clock position. - /// Index of the render. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void LogRenderBlock(MediaBlock block, TimeSpan clockPosition, int renderIndex) - { - // Prevent logging for production use - if (Platform.IsInDebugMode == false) return; - - try - { - var drift = TimeSpan.FromTicks(clockPosition.Ticks - block.StartTime.Ticks); - this.LogTrace(Aspects.RenderingWorker, - $"{block.MediaType.ToString().Substring(0, 1)} " - + $"BLK: {block.StartTime.Format()} | " - + $"CLK: {clockPosition.Format()} | " - + $"DFT: {drift.TotalMilliseconds,4:0} | " - + $"IX: {renderIndex,3} | " - + $"PQ: {Container?.Components[block.MediaType]?.BufferLength / 1024d,7:0.0}k | " - + $"TQ: {Container?.Components.BufferLength / 1024d,7:0.0}k"); - } - catch - { - // swallow - } - } - #endregion } } diff --git a/Unosquare.FFME.Common/MediaEngine.Workers.cs b/Unosquare.FFME.Common/MediaEngine.Workers.cs index d39442ab0..4b1c12baa 100644 --- a/Unosquare.FFME.Common/MediaEngine.Workers.cs +++ b/Unosquare.FFME.Common/MediaEngine.Workers.cs @@ -397,6 +397,37 @@ private bool AddNextBlock(MediaType t) return block != null; } + /// + /// Logs a block rendering operation as a Trace Message + /// if the debugger is attached. + /// + /// The block. + /// The clock position. + /// Index of the render. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void LogRenderBlock(MediaBlock block, TimeSpan clockPosition, int renderIndex) + { + // Prevent logging for production use + if (Platform.IsInDebugMode == false) return; + + try + { + var drift = TimeSpan.FromTicks(clockPosition.Ticks - block.StartTime.Ticks); + this.LogTrace(Aspects.RenderingWorker, + $"{block.MediaType.ToString().Substring(0, 1)} " + + $"BLK: {block.StartTime.Format()} | " + + $"CLK: {clockPosition.Format()} | " + + $"DFT: {drift.TotalMilliseconds,4:0} | " + + $"IX: {renderIndex,3} | " + + $"PQ: {Container?.Components[block.MediaType]?.BufferLength / 1024d,7:0.0}k | " + + $"TQ: {Container?.Components.BufferLength / 1024d,7:0.0}k"); + } + catch + { + // swallow + } + } + #endregion } } diff --git a/Unosquare.FFME.Common/Primitives/MediaBlockBuffer.cs b/Unosquare.FFME.Common/Primitives/MediaBlockBuffer.cs index c5eda1293..640979104 100644 --- a/Unosquare.FFME.Common/Primitives/MediaBlockBuffer.cs +++ b/Unosquare.FFME.Common/Primitives/MediaBlockBuffer.cs @@ -45,6 +45,10 @@ public sealed class MediaBlockBuffer : IDisposable private bool m_IsFull; private bool m_IsDisposed; + // Fast Last Lookup. + private TimeSpan LastLookupTime = TimeSpan.MinValue; + private int LastLookupIndex = -1; + #endregion #region Constructor @@ -314,45 +318,14 @@ public int IndexOf(TimeSpan renderTime) { lock (SyncLock) { - var blockCount = PlaybackBlocks.Count; - - // fast condition checking - if (blockCount <= 0) return -1; - if (blockCount == 1) return 0; - - // variable setup - var lowIndex = 0; - var highIndex = blockCount - 1; - var midIndex = 1 + lowIndex + ((highIndex - lowIndex) / 2); + if (LastLookupTime != TimeSpan.MinValue && renderTime.Ticks == LastLookupTime.Ticks) + return LastLookupIndex; - // edge condition checking - if (PlaybackBlocks[lowIndex].StartTime >= renderTime) return lowIndex; - if (PlaybackBlocks[highIndex].StartTime <= renderTime) return highIndex; + LastLookupTime = renderTime; + LastLookupIndex = PlaybackBlocks.Count > 0 && renderTime.Ticks <= PlaybackBlocks[0].StartTime.Ticks ? 0 : + PlaybackBlocks.StartIndexOf(LastLookupTime); - // First guess, very low cost, very fast - if (midIndex < highIndex - && renderTime >= PlaybackBlocks[midIndex].StartTime - && renderTime < PlaybackBlocks[midIndex + 1].StartTime) - return midIndex; - - // binary search - while (highIndex - lowIndex > 1) - { - midIndex = lowIndex + ((highIndex - lowIndex) / 2); - if (renderTime < PlaybackBlocks[midIndex].StartTime) - highIndex = midIndex; - else - lowIndex = midIndex; - } - - // linear search - for (var i = highIndex; i >= lowIndex; i--) - { - if (PlaybackBlocks[i].StartTime <= renderTime) - return i; - } - - return -1; + return LastLookupIndex; } } @@ -535,6 +508,9 @@ private static MediaBlock CreateBlock(MediaType mediaType) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateCollectionProperties() { + LastLookupIndex = -1; + LastLookupTime = TimeSpan.MinValue; + m_Count = PlaybackBlocks.Count; m_RangeStartTime = PlaybackBlocks.Count == 0 ? TimeSpan.Zero : PlaybackBlocks[0].StartTime; m_RangeEndTime = PlaybackBlocks.Count == 0 ? TimeSpan.Zero : PlaybackBlocks[PlaybackBlocks.Count - 1].EndTime; diff --git a/Unosquare.FFME.Common/Shared/Extensions.cs b/Unosquare.FFME.Common/Shared/Extensions.cs index 84d206117..0a0f30288 100644 --- a/Unosquare.FFME.Common/Shared/Extensions.cs +++ b/Unosquare.FFME.Common/Shared/Extensions.cs @@ -264,7 +264,6 @@ public static T Clamp(this T value, T min, T max) where T : struct, IComparable { if (value.CompareTo(min) < 0) return min; - return value.CompareTo(max) > 0 ? max : value; } @@ -272,6 +271,53 @@ public static T Clamp(this T value, T min, T max) #region Faster-than-Linq replacements + /// + /// Finds the index of the item that is on or greater than the specified search value + /// + /// The generic collection type + /// The value type to compare to + /// The items. + /// The value. + /// The find index. Returns -1 if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int StartIndexOf(this IList items, V value) + where T : IComparable + { + var itemCount = items.Count; + + // fast condition checking + if (itemCount <= 0) return -1; + if (itemCount == 1) return 0; + + // variable setup + var lowIndex = 0; + var highIndex = itemCount - 1; + var midIndex = 1 + lowIndex + ((highIndex - lowIndex) / 2); + + // edge condition checking + if (items[lowIndex].CompareTo(value) >= 0) return -1; + if (items[highIndex].CompareTo(value) <= 0) return highIndex; + + // binary search + while (highIndex - lowIndex > 1) + { + midIndex = lowIndex + ((highIndex - lowIndex) / 2); + if (items[midIndex].CompareTo(value) > 0) + highIndex = midIndex; + else + lowIndex = midIndex; + } + + // linear search + for (var i = highIndex; i >= lowIndex; i--) + { + if (items[i].CompareTo(value) <= 0) + return i; + } + + return -1; + } + /// /// Gets the for the main media type of the specified media container. /// diff --git a/Unosquare.FFME.Common/Shared/MediaBlock.cs b/Unosquare.FFME.Common/Shared/MediaBlock.cs index 92c6f5a9a..dbc616d6d 100644 --- a/Unosquare.FFME.Common/Shared/MediaBlock.cs +++ b/Unosquare.FFME.Common/Shared/MediaBlock.cs @@ -11,7 +11,7 @@ /// Reuse blocks as much as possible. Once you create a block from a frame, /// you don't need the frame anymore so make sure you dispose the frame. /// - public abstract class MediaBlock : IComparable, IDisposable + public abstract class MediaBlock : IComparable, IComparable, IDisposable { private readonly object SyncLock = new object(); private readonly ISyncLocker Locker = SyncLockerFactory.Create(useSlim: true); @@ -160,19 +160,19 @@ public bool Contains(TimeSpan position) && position.Ticks <= EndTime.Ticks; } - /// - /// Compares the current instance with another object of the same type and returns an integer that indicates whether the current instance precedes, follows, or occurs in the same position in the sort order as the other object. - /// - /// An object to compare with this instance. - /// - /// A value that indicates the relative order of the objects being compared. The return value has these meanings: Value Meaning Less than zero This instance precedes in the sort order. Zero This instance occurs in the same position in the sort order as . Greater than zero This instance follows in the sort order. - /// + /// public int CompareTo(MediaBlock other) { if (other == null) throw new ArgumentNullException(nameof(other)); return StartTime.Ticks.CompareTo(other.StartTime.Ticks); } + /// + public int CompareTo(TimeSpan other) + { + return StartTime.Ticks.CompareTo(other.Ticks); + } + /// public void Dispose() => Dispose(true); diff --git a/Unosquare.FFME.Common/Shared/MediaOptions.cs b/Unosquare.FFME.Common/Shared/MediaOptions.cs index c2d7e4bd9..c31dc381b 100644 --- a/Unosquare.FFME.Common/Shared/MediaOptions.cs +++ b/Unosquare.FFME.Common/Shared/MediaOptions.cs @@ -81,6 +81,13 @@ internal MediaOptions() /// public StreamInfo VideoStream { get; set; } + /// + /// Gets or sets the video seek index. + /// Use and set this + /// field while loading the options. + /// + public VideoSeekIndex VideoSeekIndex { get; set; } + /// /// Allows for a custom audio filter string. /// Please see: https://ffmpeg.org/ffmpeg-filters.html#Audio-Filters diff --git a/Unosquare.FFME.Common/Shared/VideoSeekIndex.cs b/Unosquare.FFME.Common/Shared/VideoSeekIndex.cs new file mode 100644 index 000000000..637e83742 --- /dev/null +++ b/Unosquare.FFME.Common/Shared/VideoSeekIndex.cs @@ -0,0 +1,253 @@ +namespace Unosquare.FFME.Shared +{ + using Decoding; + using FFmpeg.AutoGen; + using System; + using System.Collections.Generic; + using System.IO; + using System.Text; + + /// + /// Provides a collection of . + /// Seek entries are contain specific positions where key frames (or I frames) are located + /// within a seekable stream. + /// + public sealed class VideoSeekIndex + { + private const string VersionPrefix = "FILE-SECTION-V01"; + private static readonly string SectionHeaderText = $"{VersionPrefix}:{nameof(VideoSeekIndex)}.{nameof(Entries)}"; + private static readonly string SectionHeaderFields = $"{nameof(StreamIndex)},{nameof(SourceUrl)}"; + private static readonly string SectionDataText = $"{VersionPrefix}:{nameof(VideoSeekIndex)}.{nameof(Entries)}"; + private static readonly string SectionDataFields = + $"{nameof(VideoSeekIndexEntry.StreamIndex)}" + + $",{nameof(VideoSeekIndexEntry.StreamTimeBase)}Num" + + $",{nameof(VideoSeekIndexEntry.StreamTimeBase)}Den" + + $",{nameof(VideoSeekIndexEntry.StartTime)}" + + $",{nameof(VideoSeekIndexEntry.PresentationTime)}" + + $",{nameof(VideoSeekIndexEntry.DecodingTime)}"; + + private readonly object SyncLock = new object(); + private readonly VideoSeekIndexEntryComparer LookupComparer = new VideoSeekIndexEntryComparer(); + + /// + /// Initializes a new instance of the class. + /// + /// The source URL. + /// Index of the stream. + public VideoSeekIndex(string sourceUrl, int streamIndex) + { + SourceUrl = sourceUrl; + StreamIndex = streamIndex; + } + + /// + /// Provides access to the seek entries. + /// + public List Entries { get; } = new List(2048); + + /// + /// Gets the stream index this seeking index belongs to. + /// + public int StreamIndex { get; internal set; } + + /// + /// Gets the source URL this seeking index belongs to. + /// + public string SourceUrl { get; internal set; } + + /// + /// Loads the specified stream in the CSV-like UTF8 format it was written by the method. + /// + /// The stream. + /// The loaded index from the specified stream. + public static VideoSeekIndex Load(Stream stream) + { + var separator = new[] { ',' }; + var trimQuotes = new[] { '"' }; + var result = new VideoSeekIndex(null, -1); + + using (var reader = new StreamReader(stream, Encoding.UTF8, true)) + { + var state = 0; + + while (reader.EndOfStream == false) + { + var line = reader.ReadLine()?.Trim() ?? string.Empty; + if (state == 0 && SectionHeaderText.Equals(line)) + { + state = 1; + continue; + } + + if (state == 1 && SectionHeaderFields.Equals(line)) + { + state = 2; + continue; + } + + if (state == 2 && string.IsNullOrWhiteSpace(line) == false) + { + var parts = line.Split(separator, 2); + if (parts.Length >= 2) + { + if (int.TryParse(parts[0], out var index)) + result.StreamIndex = index; + + result.SourceUrl = parts[1].Trim(trimQuotes).Replace("\"\"", "\""); + } + + state = 3; + } + + if (state == 3 && SectionDataText.Equals(line)) + { + state = 4; + continue; + } + + if (state == 4 && SectionDataFields.Equals(line)) + { + state = 5; + continue; + } + + if (state == 5 && string.IsNullOrWhiteSpace(line) == false) + { + if (VideoSeekIndexEntry.FromCsvString(line) is VideoSeekIndexEntry entry) + result.Entries.Add(entry); + } + } + } + + return result; + } + + /// + /// Writes the index data to the specified stream in CSV-like UTF8 text format. + /// + /// The stream to write data to. + public void Save(Stream stream) + { + using (var writer = new StreamWriter(stream, Encoding.UTF8, 4096, true)) + { + writer.WriteLine(SectionHeaderText); + writer.WriteLine(SectionHeaderFields); + writer.WriteLine($"{StreamIndex},\"{SourceUrl?.Replace("\"", "\"\"")}\""); + + writer.WriteLine(SectionDataText); + writer.WriteLine(SectionDataFields); + foreach (var entry in Entries) writer.WriteLine(entry.ToCsvString()); + } + } + + /// + /// Finds the closest seek entry that is on or prior to the seek target. + /// + /// The seek target. + /// The seek entry or null of not found + public VideoSeekIndexEntry Find(TimeSpan seekTarget) + { + var index = Entries.StartIndexOf(seekTarget); + if (index < 0) return null; + return Entries[index]; + } + + /// + /// Tries to add an entry created from the frame. + /// + /// The managed frame. + /// + /// True if the index entry was created from the frame. + /// False if the frame is of wrong picture type or if it already existed. + /// + internal bool TryAdd(VideoFrame managedFrame) + { + // Update the Seek index + if (managedFrame.PictureType != AVPictureType.AV_PICTURE_TYPE_I) + return false; + + // Create the seek entry + var seekEntry = new VideoSeekIndexEntry(managedFrame); + + // Check if the entry already exists. + if (Entries.BinarySearch(seekEntry, LookupComparer) >= 0) + return false; + + // Add the seek entry and ensure they are sorted. + Entries.Add(seekEntry); + Entries.Sort(LookupComparer); + return true; + } + + /// + /// Adds the monotonic entries up to a stream duration. + /// + /// Duration of the stream. + internal void AddMonotonicEntries(TimeSpan streamDuration) + { + if (Entries.Count < 2) return; + + while (true) + { + var lastEntry = Entries[Entries.Count - 1]; + var prevEntry = Entries[Entries.Count - 2]; + + var presentationTime = lastEntry.PresentationTime == ffmpeg.AV_NOPTS_VALUE ? + ffmpeg.AV_NOPTS_VALUE : + lastEntry.PresentationTime + (lastEntry.PresentationTime - prevEntry.PresentationTime); + + var decodingTime = lastEntry.DecodingTime == ffmpeg.AV_NOPTS_VALUE ? + ffmpeg.AV_NOPTS_VALUE : + lastEntry.DecodingTime + (lastEntry.DecodingTime - prevEntry.DecodingTime); + + var startTimeTicks = lastEntry.StartTime.Ticks + (lastEntry.StartTime.Ticks - prevEntry.StartTime.Ticks); + + var entry = new VideoSeekIndexEntry( + lastEntry.StreamIndex, + lastEntry.StreamTimeBase.num, + lastEntry.StreamTimeBase.den, + startTimeTicks, + presentationTime, + decodingTime); + + if (entry.StartTime.Ticks > streamDuration.Ticks) + return; + + Entries.Add(entry); + } + } + + /// + /// Gets the monotonic presentation distance units that separate the last entries in the index. + /// Returns -1 if there are less than 2 entries or if the entries are not monotonic. + /// + /// -1 if the entries are not monotonic + internal long ComputeMonotonicDistance() + { + if (Entries.Count < 2) return -1L; + var lastDistance = -1L; + var currentDistance = -1L; + + for (var i = Entries.Count - 1; i > 0; i--) + { + currentDistance = Entries[i].PresentationTime - Entries[i - 1].PresentationTime; + if (lastDistance != -1L && lastDistance != currentDistance) + return -1L; + + lastDistance = currentDistance; + } + + return currentDistance; + } + + /// + /// A comparer for + /// + private class VideoSeekIndexEntryComparer : IComparer + { + /// + public int Compare(VideoSeekIndexEntry x, VideoSeekIndexEntry y) => + x.StartTime.Ticks.CompareTo(y.StartTime.Ticks); + } + } +} diff --git a/Unosquare.FFME.Common/Shared/VideoSeekIndexEntry.cs b/Unosquare.FFME.Common/Shared/VideoSeekIndexEntry.cs new file mode 100644 index 000000000..a2f675afc --- /dev/null +++ b/Unosquare.FFME.Common/Shared/VideoSeekIndexEntry.cs @@ -0,0 +1,117 @@ +namespace Unosquare.FFME.Shared +{ + using Decoding; + using FFmpeg.AutoGen; + using System; + + /// + /// Represents a seek entry to a position within the stream + /// + public sealed class VideoSeekIndexEntry : IComparable, IComparable + { + private static readonly char[] CommaSeparator = new[] { ',' }; + + /// + /// Initializes a new instance of the class. + /// + /// Index of the stream. + /// The time base numerator. + /// The time base deonominator. + /// The start time ticks. + /// The presentation time. + /// The decoding time. + internal VideoSeekIndexEntry(int streamIndex, int timeBaseNum, int timeBaseDen, long startTimeTicks, long presentationTime, long decodingTime) + { + StreamIndex = streamIndex; + StartTime = TimeSpan.FromTicks(startTimeTicks); + PresentationTime = presentationTime; + DecodingTime = decodingTime; + StreamTimeBase = new AVRational { num = timeBaseNum, den = timeBaseDen }; + } + + /// + /// Initializes a new instance of the class. + /// + /// The frame. + internal VideoSeekIndexEntry(VideoFrame frame) + { + StreamIndex = frame.StreamIndex; + StreamTimeBase = frame.StreamTimeBase; + StartTime = frame.StartTime; + PresentationTime = frame.PresentationTime; + DecodingTime = frame.DecodingTime; + } + + /// + /// Gets the stream index of this index entry. + /// + public int StreamIndex { get; } + + /// + /// Gets the stream time base. + /// + public AVRational StreamTimeBase { get; } + + /// + /// Gets the start time of the frame. + /// + public TimeSpan StartTime { get; } + + /// + /// Gets the original, unadjusted presentation time. + /// + public long PresentationTime { get; } + + /// + /// Gets the original, unadjusted decoding time. + /// + public long DecodingTime { get; } + + /// + public int CompareTo(VideoSeekIndexEntry other) + { + if (other == null) throw new ArgumentNullException(nameof(other)); + return StartTime.Ticks.CompareTo(other.StartTime.Ticks); + } + + /// + public int CompareTo(TimeSpan other) + { + return StartTime.Ticks.CompareTo(other.Ticks); + } + + /// + public override string ToString() => + $"IX: {StreamIndex} | TB: {StreamTimeBase.num}/{StreamTimeBase.den} | ST: {StartTime.Format()} | PTS: {PresentationTime} | DTS: {DecodingTime}"; + + /// + /// Creates an entry based on a CSV string + /// + /// The line. + /// An index entry or null if unsuccessful. + internal static VideoSeekIndexEntry FromCsvString(string line) + { + var parts = line.Split(CommaSeparator); + if (parts.Length >= 6 && + int.TryParse(parts[0], out var streamIndex) && + int.TryParse(parts[1], out var timeBaseNum) && + int.TryParse(parts[2], out var timeBaseDen) && + long.TryParse(parts[3], out var startTimeTicks) && + long.TryParse(parts[4], out var presentationTime) && + long.TryParse(parts[5], out var decodingTime)) + { + return new VideoSeekIndexEntry( + streamIndex, timeBaseNum, timeBaseDen, startTimeTicks, presentationTime, decodingTime); + } + + return null; + } + + /// + /// Converts values of this instance to a line of CSV text. + /// + /// The comma-separated values + internal string ToCsvString() => + $"{StreamIndex},{StreamTimeBase.num},{StreamTimeBase.den},{StartTime.Ticks},{PresentationTime},{DecodingTime}"; + } +} diff --git a/Unosquare.FFME.Windows.Sample/MainWindow.MediaEvents.cs b/Unosquare.FFME.Windows.Sample/MainWindow.MediaEvents.cs index 5db3f6dfb..1bb9d6a3e 100644 --- a/Unosquare.FFME.Windows.Sample/MainWindow.MediaEvents.cs +++ b/Unosquare.FFME.Windows.Sample/MainWindow.MediaEvents.cs @@ -112,22 +112,27 @@ private void OnMediaInitializing(object sender, MediaInitializingEventArgs e) /// The instance containing the event data. private void OnMediaOpening(object sender, MediaOpeningEventArgs e) { + const string SideLoadAspect = "Client.SideLoad"; + // You can start off by adjusting subtitles delay // e.Options.SubtitlesDelay = TimeSpan.FromSeconds(7); // See issue #216 - // Example of automatically side-loading SRT subs + // Get the local file path from the URL (if possible) + var mediaFilePath = string.Empty; try { - var inputUrl = e.Info.InputUrl; - var url = new Uri(inputUrl); - if (url.IsFile || url.IsUnc) - { - inputUrl = Path.ChangeExtension(url.LocalPath, "srt"); - if (File.Exists(inputUrl)) - e.Options.SubtitlesUrl = inputUrl; - } + var url = new Uri(e.Info.InputUrl); + mediaFilePath = url.IsFile || url.IsUnc ? Path.GetFullPath(url.LocalPath) : string.Empty; + } + catch { /* Ignore Exceptions */ } + + // Example of automatically side-loading SRT subs + if (string.IsNullOrWhiteSpace(mediaFilePath) == false) + { + var srtFilePath = Path.ChangeExtension(mediaFilePath, "srt"); + if (File.Exists(srtFilePath)) + e.Options.SubtitlesUrl = srtFilePath; } - catch { /* Ignore exception and continue */ } // You can force video FPS if necessary // see: https://github.com/unosquare/ffmediaelement/issues/212 @@ -153,6 +158,32 @@ private void OnMediaOpening(object sender, MediaOpeningEventArgs e) // ReSharper disable once InvertIf if (e.Options.VideoStream is StreamInfo videoStream) { + // If we have a valid seek index let's use it! + if (string.IsNullOrWhiteSpace(mediaFilePath) == false) + { + try + { + // Try to Create or Load a Seek Index + var durationSeconds = e.Info.Duration.TotalSeconds > 0 ? e.Info.Duration.TotalSeconds : 0; + var seekIndex = LoadOrCreateVideoSeekIndex(mediaFilePath, videoStream.StreamIndex, durationSeconds); + + // Make sure the seek index belongs to the media file path + if (seekIndex != null && + string.IsNullOrWhiteSpace(seekIndex.SourceUrl) == false && + seekIndex.SourceUrl.Equals(mediaFilePath) && + seekIndex.StreamIndex == videoStream.StreamIndex) + { + // Set the index on the options object. + e.Options.VideoSeekIndex = seekIndex; + } + } + catch (Exception ex) + { + // Log the exception, and ignore it. Continue execution. + Media?.LogError(SideLoadAspect, "Error loading seek index data.", ex); + } + } + // Hardware device priorities var deviceCandidates = new[] { @@ -313,7 +344,7 @@ private async void OnAudioDeviceStopped(object sender, EventArgs e) #endregion - #region Other Media Event Handlers + #region Other Media Event Handlers and Methods /// /// Handles the PositionChanged event of the Media control. @@ -325,6 +356,41 @@ private void OnMediaPositionChanged(object sender, PositionChangedRoutedEventArg // Handle position change notifications } + /// + /// Loads the index of the or create media seek. + /// + /// The URL. + /// The associated stream index. + /// The duration in seconds. + /// + /// The seek index + /// + private VideoSeekIndex LoadOrCreateVideoSeekIndex(string mediaFilePath, int streamIndex, double durationSeconds) + { + var seekFileName = $"{Path.GetFileNameWithoutExtension(mediaFilePath)}.six"; + var seekFilePath = Path.Combine(App.Current.ViewModel.Playlist.IndexDirectory, seekFileName); + if (string.IsNullOrWhiteSpace(seekFilePath)) return null; + + if (File.Exists(seekFilePath)) + { + using (var stream = File.OpenRead(seekFilePath)) + return VideoSeekIndex.Load(stream); + } + else + { + if (GuiContext.Current.IsInDebugMode == false || durationSeconds <= 0 || durationSeconds >= 60) + return null; + + var seekIndex = MediaEngine.CreateVideoSeekIndex(mediaFilePath, streamIndex); + if (seekIndex.Entries.Count <= 0) return null; + + using (var stream = File.OpenWrite(seekFilePath)) + seekIndex.Save(stream); + + return seekIndex; + } + } + #endregion } } \ No newline at end of file diff --git a/Unosquare.FFME.Windows.Sample/Properties/AssemblyInfo.cs b/Unosquare.FFME.Windows.Sample/Properties/AssemblyInfo.cs index 35bffe5a9..df7444c3f 100644 --- a/Unosquare.FFME.Windows.Sample/Properties/AssemblyInfo.cs +++ b/Unosquare.FFME.Windows.Sample/Properties/AssemblyInfo.cs @@ -40,5 +40,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2018.09.16.*")] +[assembly: AssemblyVersion("2018.09.23.*")] [assembly: AssemblyFileVersion("4.0.270")] diff --git a/Unosquare.FFME.Windows.Sample/ViewModels/PlaylistViewModel.cs b/Unosquare.FFME.Windows.Sample/ViewModels/PlaylistViewModel.cs index a649ca676..9ebeadf76 100644 --- a/Unosquare.FFME.Windows.Sample/ViewModels/PlaylistViewModel.cs +++ b/Unosquare.FFME.Windows.Sample/ViewModels/PlaylistViewModel.cs @@ -46,6 +46,11 @@ public PlaylistViewModel(RootViewModel root) if (Directory.Exists(ThumbsDirectory) == false) Directory.CreateDirectory(ThumbsDirectory); + // Set and create a index directory + IndexDirectory = Path.Combine(root.AppDataDirectory, "SeekIndexes"); + if (Directory.Exists(IndexDirectory) == false) + Directory.CreateDirectory(IndexDirectory); + PlaylistFilePath = Path.Combine(root.AppDataDirectory, "ffme.m3u8"); Entries = new CustomPlaylist(this); @@ -82,6 +87,11 @@ public PlaylistViewModel(RootViewModel root) /// public string ThumbsDirectory { get; } + /// + /// Gets the seek index base directory. + /// + public string IndexDirectory { get; } + /// /// Gets the playlist file path. /// diff --git a/Unosquare.FFME.Windows/Properties/AssemblyInfo.cs b/Unosquare.FFME.Windows/Properties/AssemblyInfo.cs index d875754fa..2756811d0 100644 --- a/Unosquare.FFME.Windows/Properties/AssemblyInfo.cs +++ b/Unosquare.FFME.Windows/Properties/AssemblyInfo.cs @@ -31,5 +31,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2018.09.16.*")] +[assembly: AssemblyVersion("2018.09.23.*")] [assembly: AssemblyFileVersion("4.0.270")]