From 96edc17b2f1cf4e92e0cda2a729b69dd226e655c Mon Sep 17 00:00:00 2001 From: riina <54872398+riina@users.noreply.github.com> Date: Sun, 19 Mar 2023 03:01:39 -0700 Subject: [PATCH] add support for using top-down during main save --- src/Art.M3U/IExtraSaverOperation.cs | 24 +++++ src/Art.M3U/M3UDownloaderContext.cs | 3 +- src/Art.M3U/M3UDownloaderContextProcessor.cs | 46 +++++++-- .../M3UDownloaderContextStandardSaver.cs | 6 +- .../M3UDownloaderContextStreamOutputSaver.cs | 2 +- .../M3UDownloaderContextTopDownSaver.cs | 97 +++++++++++++------ 6 files changed, 134 insertions(+), 44 deletions(-) create mode 100644 src/Art.M3U/IExtraSaverOperation.cs diff --git a/src/Art.M3U/IExtraSaverOperation.cs b/src/Art.M3U/IExtraSaverOperation.cs new file mode 100644 index 0000000..18deeda --- /dev/null +++ b/src/Art.M3U/IExtraSaverOperation.cs @@ -0,0 +1,24 @@ +namespace Art.M3U; + +/// +/// Represents an extra operation that should be used when no new segments are immediately available in . +/// +/// +/// The method can be invoked multiple times, and +/// is intended to be used when no new segments are immediately available. +/// +public interface IExtraSaverOperation +{ + /// + /// Resets this operation. + /// + void Reset(); + + /// + /// Executes operation step. + /// + /// Existing M3U file. + /// Cancellation token. + /// Task that returns false if this operation is no longer useful. + Task TickAsync(M3UFile m3, CancellationToken cancellationToken = default); +} diff --git a/src/Art.M3U/M3UDownloaderContext.cs b/src/Art.M3U/M3UDownloaderContext.cs index caa7826..682b19b 100644 --- a/src/Art.M3U/M3UDownloaderContext.cs +++ b/src/Art.M3U/M3UDownloaderContext.cs @@ -164,8 +164,9 @@ private async Task WriteAncillaryFileAsync(string file, ReadOnlyMemory dat /// /// If true, only request target once. /// Timeout when waiting for new entries. + /// Extra operation to invoke during down time. /// Downloader. - public M3UDownloaderContextStandardSaver CreateStandardSaver(bool oneOff, TimeSpan timeout) => new(this, oneOff, timeout); + public M3UDownloaderContextStandardSaver CreateStandardSaver(bool oneOff, TimeSpan timeout, IExtraSaverOperation? extraOperation = null) => new(this, oneOff, timeout, extraOperation); /// /// Creates a basic stream output downloader. diff --git a/src/Art.M3U/M3UDownloaderContextProcessor.cs b/src/Art.M3U/M3UDownloaderContextProcessor.cs index 4199331..2314fa5 100644 --- a/src/Art.M3U/M3UDownloaderContextProcessor.cs +++ b/src/Art.M3U/M3UDownloaderContextProcessor.cs @@ -109,9 +109,11 @@ private static async Task DelayOrThrowAsync(ArtHttpResponseMessageException exce /// If true, complete after one pass through playlist. /// Timeout to use to determine when a stream seems to have ended. /// Processor to handle playlist elements. + /// Optional extra operation. /// Cancellation token. - protected async Task ProcessPlaylistAsync(bool oneOff, TimeSpan timeout, IPlaylistElementProcessor playlistElementProcessor, CancellationToken cancellationToken = default) + protected async Task ProcessPlaylistAsync(bool oneOff, TimeSpan timeout, IPlaylistElementProcessor playlistElementProcessor, IExtraSaverOperation? extraOperation = null, CancellationToken cancellationToken = default) { + extraOperation?.Reset(); IOperationProgressContext? operationProgressContext = null; try { @@ -174,21 +176,47 @@ protected async Task ProcessPlaylistAsync(bool oneOff, TimeSpan timeout, IPlayli if (j != 0) { sw.Restart(); + remainingTimeout = timeout; } else if (sw.IsRunning) { - var elapsed = sw.Elapsed; - if (elapsed >= timeout) + if (extraOperation != null) { - if (operationProgressContext != null) + try { - operationProgressContext.Dispose(); - operationProgressContext = null; + Context.Tool.LogInformation("No new segments, executing extra operation..."); + bool shouldContinue = await extraOperation.TickAsync(m3, cancellationToken).ConfigureAwait(false); + if (!shouldContinue) + { + extraOperation = null; + } } - Context.Tool.LogInformation($"No new entries for timeout {timeout}, stopping"); - return; + catch (Exception e) + { + Context.Tool.LogError(e.Message, e.ToString()); + extraOperation = null; + } + } + if (extraOperation != null) + { + sw.Restart(); + remainingTimeout = timeout; + } + else + { + var elapsed = sw.Elapsed; + if (elapsed >= timeout) + { + if (operationProgressContext != null) + { + operationProgressContext.Dispose(); + operationProgressContext = null; + } + Context.Tool.LogInformation($"No new entries for timeout {timeout}, stopping"); + return; + } + remainingTimeout = timeout.Subtract(elapsed); } - remainingTimeout = timeout.Subtract(elapsed); } else { diff --git a/src/Art.M3U/M3UDownloaderContextStandardSaver.cs b/src/Art.M3U/M3UDownloaderContextStandardSaver.cs index d9ed8a6..ac2fad7 100644 --- a/src/Art.M3U/M3UDownloaderContextStandardSaver.cs +++ b/src/Art.M3U/M3UDownloaderContextStandardSaver.cs @@ -7,16 +7,18 @@ public class M3UDownloaderContextStandardSaver : M3UDownloaderContextSaver { private readonly bool _oneOff; private readonly TimeSpan _timeout; + private readonly IExtraSaverOperation? _extraOperation; - internal M3UDownloaderContextStandardSaver(M3UDownloaderContext context, bool oneOff, TimeSpan timeout) : base(context) + internal M3UDownloaderContextStandardSaver(M3UDownloaderContext context, bool oneOff, TimeSpan timeout, IExtraSaverOperation? extraOperation) : base(context) { _oneOff = oneOff; _timeout = timeout; + _extraOperation = extraOperation; } /// public override Task RunAsync(CancellationToken cancellationToken = default) { - return ProcessPlaylistAsync(_oneOff, _timeout, new SegmentDownloadPlaylistElementProcessor(Context), cancellationToken); + return ProcessPlaylistAsync(_oneOff, _timeout, new SegmentDownloadPlaylistElementProcessor(Context), _extraOperation, cancellationToken); } } diff --git a/src/Art.M3U/M3UDownloaderContextStreamOutputSaver.cs b/src/Art.M3U/M3UDownloaderContextStreamOutputSaver.cs index a5f9f61..03753a4 100644 --- a/src/Art.M3U/M3UDownloaderContextStreamOutputSaver.cs +++ b/src/Art.M3U/M3UDownloaderContextStreamOutputSaver.cs @@ -26,6 +26,6 @@ internal M3UDownloaderContextStreamOutputSaver(M3UDownloaderContext context, boo /// Thrown on HTTP response indicating non-successful response. public Task ExportAsync(Stream stream, CancellationToken cancellationToken = default) { - return ProcessPlaylistAsync(_oneOff, _timeout, new StreamOutputPlaylistElementProcessor(Context, stream), cancellationToken); + return ProcessPlaylistAsync(_oneOff, _timeout, new StreamOutputPlaylistElementProcessor(Context, stream), null, cancellationToken); } } diff --git a/src/Art.M3U/M3UDownloaderContextTopDownSaver.cs b/src/Art.M3U/M3UDownloaderContextTopDownSaver.cs index 60f71f0..9262b41 100644 --- a/src/Art.M3U/M3UDownloaderContextTopDownSaver.cs +++ b/src/Art.M3U/M3UDownloaderContextTopDownSaver.cs @@ -8,7 +8,7 @@ namespace Art.M3U; /// /// Represents a top-down saver. /// -public partial class M3UDownloaderContextTopDownSaver : M3UDownloaderContextSaver +public partial class M3UDownloaderContextTopDownSaver : M3UDownloaderContextSaver, IExtraSaverOperation { [GeneratedRegex("(^[\\S\\s]*[^\\d]|)\\d+(\\.\\w+)$")] private static partial Regex GetBitRegex(); @@ -18,6 +18,8 @@ public partial class M3UDownloaderContextTopDownSaver : M3UDownloaderContextSave private readonly long _top; private readonly Func _nameTransform; + private long _currentTop; + private bool _ended; internal M3UDownloaderContextTopDownSaver(M3UDownloaderContext context, long top) : this(context, top, TranslateNameDefault) @@ -32,6 +34,7 @@ internal M3UDownloaderContextTopDownSaver(M3UDownloaderContext context, long top internal M3UDownloaderContextTopDownSaver(M3UDownloaderContext context, long top, Func nameTransform) : base(context) { _top = top; + _currentTop = _top; _nameTransform = nameTransform; } @@ -81,45 +84,77 @@ public static string TranslateNameMatchLength(string name, string i) /// public override async Task RunAsync(CancellationToken cancellationToken = default) { - FailCounter = 0; - long top = _top; + Reset(); while (true) { + if (HeartbeatCallback != null) await HeartbeatCallback().ConfigureAwait(false); + M3UFile m3; + Context.Tool.LogInformation("Reading main..."); try { - if (top < 0) break; - if (HeartbeatCallback != null) await HeartbeatCallback().ConfigureAwait(false); - Context.Tool.LogInformation("Reading main..."); - M3UFile m3 = await Context.GetAsync(cancellationToken).ConfigureAwait(false); - string str = m3.DataLines.First(); - Uri origUri = new(Context.MainUri, str); - int idx = str.IndexOf('?'); - if (idx >= 0) str = str[..idx]; - Uri uri = new UriBuilder(new Uri(Context.MainUri, _nameTransform(str, top))) { Query = origUri.Query }.Uri; - Context.Tool.LogInformation($"Downloading segment {uri.Segments[^1]}..."); - try - { - // Don't assume MSN, and just accept failure (exception) when trying to decrypt with no IV - // Also don't depend on current file since it probably won't do us good anyway for this use case - await Context.DownloadSegmentAsync(uri, null, null, cancellationToken).ConfigureAwait(false); - top--; - } - catch (ArtHttpResponseMessageException e) - { - if (e.StatusCode == HttpStatusCode.NotFound) - { - Context.Tool.LogInformation("HTTP NotFound returned, ending operation"); - return; - } - await HandleRequestExceptionAsync(e, cancellationToken).ConfigureAwait(false); - } - await Task.Delay(500, cancellationToken).ConfigureAwait(false); - FailCounter = 0; + m3 = await Context.GetAsync(cancellationToken).ConfigureAwait(false); } catch (ArtHttpResponseMessageException e) { await HandleRequestExceptionAsync(e, cancellationToken).ConfigureAwait(false); + continue; + } + bool shouldContinue = await TickAsync(m3, cancellationToken).ConfigureAwait(false); + if (!shouldContinue) + { + return; } + await Task.Delay(500, cancellationToken).ConfigureAwait(false); } } + + void IExtraSaverOperation.Reset() + { + Reset(); + } + + private void Reset() + { + _ended = false; + _currentTop = _top; + FailCounter = 0; + } + + Task IExtraSaverOperation.TickAsync(M3UFile m3, CancellationToken cancellationToken) + { + return TickAsync(m3, cancellationToken); + } + + private async Task TickAsync(M3UFile m3, CancellationToken cancellationToken = default) + { + if (_ended || _currentTop < 0) + { + return false; + } + string str = m3.DataLines.First(); + Uri origUri = new(Context.MainUri, str); + int idx = str.IndexOf('?'); + if (idx >= 0) str = str[..idx]; + Uri uri = new UriBuilder(new Uri(Context.MainUri, _nameTransform(str, _currentTop))) { Query = origUri.Query }.Uri; + Context.Tool.LogInformation($"Top-downloading segment {uri.Segments[^1]}..."); + try + { + // Don't assume MSN, and just accept failure (exception) when trying to decrypt with no IV + // Also don't depend on current file since it probably won't do us good anyway for this use case + await Context.DownloadSegmentAsync(uri, null, null, cancellationToken).ConfigureAwait(false); + _currentTop--; + } + catch (ArtHttpResponseMessageException e) + { + if (e.StatusCode == HttpStatusCode.NotFound) + { + Context.Tool.LogInformation("HTTP NotFound returned, ending top-down operation"); + _ended = true; + return false; + } + await HandleRequestExceptionAsync(e, cancellationToken).ConfigureAwait(false); + } + FailCounter = 0; + return true; + } }