Add -Resume Feature to Web Cmdlets (#6447)

Fixes #5964

Adds -Resume switch to Invoke-WebRequest and Invoke-RestMethod

-Resume requires -OutFile

Enables the ability to resume downloading a partially or incompletely downloaded file.

File Size is the only indicator of local and remote file parity.

If the local file is smaller than the remote file and the remote endpoint supports resume, the local file will be appended with the remaining bytes.

If the local file is larger than the remote file, the local file will be overwritten

If the remote server does not support resume, the local file will be overwritten

If the local file is the same size as the remote file, the remote endpoint will return a 416 status code. This response is special-cased as a success in this instance. The local file remains untouched and it is assumed the file was already successfully downloaded previously.

If the local file does not exist it will be created and the entire remote file will be requested.

Added tests for all code new code paths (I'm pretty sure anyway)

Added /Resume Controller to WebListener

Documented /Resume Controller

Updated .spelling to reflect terms in WebListener docs

Note: I had to change the way GetResponse() tracks the current URI as we now have 3 places where the call is taking place. I don't foresee this causing any regressions. This area needs some refactoring. especially if we want to implement a retry mechanism

.spelling

@@ -1104,6 +1104,8 @@ v6.0.
 #region test/tools/WebListener/README.md Overrides
  - test/tools/WebListener/README.md
 Auth
+NoResume
 NTLM
+NumberBytes
 ResponseHeaders
 #endregion

src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs

@@ -354,6 +354,12 @@ public virtual string CustomMethod
         [Parameter]
         public virtual SwitchParameter PassThru { get; set; }
 
+        /// <summary>
+        /// Resumes downloading a partial or incomplete file. OutFile is required.
+        /// </summary>
+        [Parameter]
+        public virtual SwitchParameter Resume { get; set; }
+
         #endregion
 
         #endregion Virtual Properties
@@ -512,7 +518,15 @@ internal virtual void ValidateParameters()
             if (PassThru && (OutFile == null))
             {
                 ErrorRecord error = GetValidationError(WebCmdletStrings.OutFileMissing,
-                                                       "WebCmdletOutFileMissingException");
+                                                       "WebCmdletOutFileMissingException", nameof(PassThru));
+                ThrowTerminatingError(error);
+            }
+
+            // Resume requires OutFile.
+            if (Resume.IsPresent && OutFile == null)
+            {
+                ErrorRecord error = GetValidationError(WebCmdletStrings.OutFileMissing,
+                                                       "WebCmdletOutFileMissingException", nameof(Resume));
                 ThrowTerminatingError(error);
             }
         }
@@ -637,6 +651,14 @@ internal bool ShouldWriteToPipeline
             get { return (!ShouldSaveToOutFile || PassThru); }
         }
 
+        /// <summary>
+        /// Determines whether writing to a file should Resume and append rather than overwrite.
+        /// </summary>
+        internal bool ShouldResume
+        {
+            get { return (Resume.IsPresent && _resumeSuccess); }
+        }
+
         #endregion Helper Properties
 
         #region Helper Methods
@@ -860,6 +882,16 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet
         /// </summary>
         internal int _maximumFollowRelLink = Int32.MaxValue;
 
+        /// <summary>
+        /// The remote endpoint returned a 206 status code indicating successful resume.
+        /// </summary>
+        private bool _resumeSuccess = false;
+
+        /// <summary>
+        /// The current size of the local file being resumed.
+        /// </summary>
+        private long _resumeFileSize = 0;
+
         private HttpMethod GetHttpMethod(WebRequestMethod method)
         {
             switch (Method)
@@ -1062,6 +1094,22 @@ internal virtual HttpRequestMessage GetRequest(Uri uri, bool stripAuthorization)
                 }
             }
 
+            // If the file to resume downloading exists, create the Range request header using the file size.
+            // If not, create a Range to request the entire file.
+            if (Resume.IsPresent)
+            {
+                var fileInfo = new FileInfo(QualifiedOutFile);
+                if (fileInfo.Exists)
+                {
+                    request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null);
+                    _resumeFileSize = fileInfo.Length;
+                }
+                else
+                {
+                    request.Headers.Range = new RangeHeaderValue(0, null);
+                }
+            }
+
             // Some web sites (e.g. Twitter) will return exception on POST when Expect100 is sent
             // Default behavior is continue to send body content anyway after a short period
             // Here it send the two part as a whole.
@@ -1233,6 +1281,9 @@ internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestM
             if (client == null) { throw new ArgumentNullException("client"); }
             if (request == null) { throw new ArgumentNullException("request"); }
 
+            // Track the current URI being used by various requests and re-requests.
+            var currentUri = request.RequestUri;
+
             _cancelToken = new CancellationTokenSource();
             HttpResponseMessage response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
 
@@ -1256,14 +1307,51 @@ internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestM
                 }
 
                 // recreate the HttpClient with redirection enabled since the first call suppressed redirection
+                currentUri = new Uri(request.RequestUri, response.Headers.Location);
                 using (client = GetHttpClient(false))
-                using (HttpRequestMessage redirectRequest = GetRequest(new Uri(request.RequestUri, response.Headers.Location), stripAuthorization:true))
+                using (HttpRequestMessage redirectRequest = GetRequest(currentUri, stripAuthorization:true))
                 {
                     FillRequestStream(redirectRequest);
                     _cancelToken = new CancellationTokenSource();
                     response = client.SendAsync(redirectRequest, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult();
                 }
             }
+
+            // Request again without the Range header because the server indicated the range was not satisfiable.
+            // This happens when the local file is larger than the remote file.
+            // If the size of the remote file is the same as the local file, there is nothing to resume.
+            if (Resume.IsPresent && 
+                response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable && 
+                (response.Content.Headers.ContentRange.HasLength &&
+                response.Content.Headers.ContentRange.Length != _resumeFileSize))
+            {
+                _cancelToken.Cancel();
+
+                WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg);
+
+                // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream()
+                // are treated as a standard -OutFile request. This also disables appending local file.
+                Resume = new SwitchParameter(false);
+
+                using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri, stripAuthorization:false))
+                {
+                    FillRequestStream(requestWithoutRange);
+                    long requestContentLength = 0;
+                    if (requestWithoutRange.Content != null)
+                        requestContentLength = requestWithoutRange.Content.Headers.ContentLength.Value;
+
+                    string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture,
+                        WebCmdletStrings.WebMethodInvocationVerboseMsg,
+                        requestWithoutRange.Method,
+                        requestWithoutRange.RequestUri,
+                        requestContentLength);
+                    WriteVerbose(reqVerboseMsg);
+
+                    return GetResponse(client, requestWithoutRange, stripAuthorization);
+                }
+            }
+
+            _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent;
             return response;
         }
 
@@ -1336,7 +1424,22 @@ protected override void ProcessRecord()
                                     contentType);
                                 WriteVerbose(respVerboseMsg);
 
-                                if (!response.IsSuccessStatusCode)
+                                bool _isSuccess = response.IsSuccessStatusCode;
+
+                                // Check if the Resume range was not satisfiable because the file already completed downloading.
+                                // This happens when the local file is the same size as the remote file.
+                                if (Resume.IsPresent &&
+                                    response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable && 
+                                    response.Content.Headers.ContentRange.HasLength && 
+                                    response.Content.Headers.ContentRange.Length == _resumeFileSize)
+                                {
+                                    _isSuccess = true;
+                                    WriteVerbose(String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.OutFileWritingSkipped, OutFile));
+                                    // Disable writing to the OutFile.
+                                    OutFile = null;
+                                }
+
+                                if (!_isSuccess)
                                 {
                                     string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure,
                                         (int)response.StatusCode, response.ReasonPhrase);

src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs

@@ -342,9 +342,20 @@ internal static void WriteToStream(byte[] input, Stream output)
         /// <param name="cmdlet"></param>
         internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet)
         {
-            using (FileStream output = File.Create(filePath))
+            // If the web cmdlet should resume, append the file instead of overwriting.
+            if(cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume)
             {
-                WriteToStream(stream, output, cmdlet);
+                using (FileStream output = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.Read))
+                {
+                    WriteToStream(stream, output, cmdlet);
+                }
+            }
+            else
+            {
+                using (FileStream output = File.Create(filePath))
+                {
+                    WriteToStream(stream, output, cmdlet);
+                }
             }
         }
 

src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx

@@ -187,7 +187,10 @@
     <value>Path '{0}' is not a file system path. Please specify the path to a file in the file system.</value>
   </data>
   <data name="OutFileMissing" xml:space="preserve">
-    <value>The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the PassThru parameter, then retry.</value>
+    <value>The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the {0} parameter, then retry.</value>
+  </data>
+  <data name="OutFileWritingSkipped" xml:space="preserve">
+    <value>The file will not be re-downloaded because the remote file is the same size as the OutFile: {0}</value>
   </data>
   <data name="ProxyCredentialConflict" xml:space="preserve">
     <value>The cmdlet cannot run because the following conflicting parameters are specified: ProxyCredential and ProxyUseDefaultCredentials. Specify either ProxyCredential or ProxyUseDefaultCredentials, then retry.</value>
@@ -249,6 +252,9 @@
   <data name="WebMethodInvocationVerboseMsg" xml:space="preserve">
     <value>{0} {1} with {2}-byte payload</value>
   </data>
+  <data name="WebMethodResumeFailedVerboseMsg" xml:space="preserve">
+    <value>The remote server indicated it could not resume downloading. The local file will be overwritten.</value>
+  </data>
   <data name="WebResponseVerboseMsg" xml:space="preserve">
     <value>received {0}-byte response of content type {1}</value>
   </data>