diff --git a/GitHub.Unity.sln.DotSettings b/GitHub.Unity.sln.DotSettings index d4374a413..31c5e56f1 100644 --- a/GitHub.Unity.sln.DotSettings +++ b/GitHub.Unity.sln.DotSettings @@ -335,6 +335,7 @@ </TypePattern> </Patterns> ID + MD SSH <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> diff --git a/src/GitHub.Api/GitHub.Api.csproj b/src/GitHub.Api/GitHub.Api.csproj index 337412647..85382dd72 100644 --- a/src/GitHub.Api/GitHub.Api.csproj +++ b/src/GitHub.Api/GitHub.Api.csproj @@ -139,6 +139,7 @@ + diff --git a/src/GitHub.Api/IO/FileSystem.cs b/src/GitHub.Api/IO/FileSystem.cs index 4c9920da3..297d7c1bf 100644 --- a/src/GitHub.Api/IO/FileSystem.cs +++ b/src/GitHub.Api/IO/FileSystem.cs @@ -34,6 +34,12 @@ public bool FileExists(string filename) return File.Exists(filename); } + public long FileLength(string path) + { + var fileInfo = new FileInfo(path); + return fileInfo.Length; + } + public IEnumerable GetDirectories(string path) { return Directory.GetDirectories(path); @@ -167,6 +173,11 @@ public void WriteAllText(string path, string contents, Encoding encoding) File.WriteAllText(path, contents, encoding); } + public byte[] ReadAllBytes(string path) + { + return File.ReadAllBytes(path); + } + public string ReadAllText(string path) { return File.ReadAllText(path); @@ -201,5 +212,10 @@ public Stream OpenRead(string path) { return File.OpenRead(path); } + + public Stream OpenWrite(string path, FileMode mode) + { + return new FileStream(path, mode); + } } } diff --git a/src/GitHub.Api/IO/IFileSystem.cs b/src/GitHub.Api/IO/IFileSystem.cs index f2e5d225e..6e78038f6 100644 --- a/src/GitHub.Api/IO/IFileSystem.cs +++ b/src/GitHub.Api/IO/IFileSystem.cs @@ -7,6 +7,7 @@ namespace GitHub.Unity public interface IFileSystem { bool FileExists(string path); + long FileLength(string path); string Combine(string path1, string path2); string Combine(string path1, string path2, string path3); string GetFullPath(string path); @@ -37,9 +38,11 @@ public interface IFileSystem string ReadAllText(string path); string ReadAllText(string path, Encoding encoding); Stream OpenRead(string path); + Stream OpenWrite(string path, FileMode mode); string[] ReadAllLines(string path); char DirectorySeparatorChar { get; } bool ExistingPathIsDirectory(string path); void SetCurrentDirectory(string currentDirectory); + byte[] ReadAllBytes(string path); } } \ No newline at end of file diff --git a/src/GitHub.Api/IO/NiceIO.cs b/src/GitHub.Api/IO/NiceIO.cs index 0b10a5f0a..ef6b91247 100644 --- a/src/GitHub.Api/IO/NiceIO.cs +++ b/src/GitHub.Api/IO/NiceIO.cs @@ -907,6 +907,12 @@ public NPath WriteAllText(string contents, Encoding encoding) return this; } + public byte[] ReadAllBytes() + { + ThrowIfRelative(); + return FileSystem.ReadAllBytes(ToString()); + } + public string ReadAllText() { ThrowIfRelative(); diff --git a/src/GitHub.Api/Tasks/DownloadTask.cs b/src/GitHub.Api/Tasks/DownloadTask.cs new file mode 100644 index 000000000..0d5bca859 --- /dev/null +++ b/src/GitHub.Api/Tasks/DownloadTask.cs @@ -0,0 +1,316 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Text; +using System.Threading; + +namespace GitHub.Unity +{ + public class Utils + { + public static bool Copy(Stream source, Stream destination, int chunkSize) + { + return Copy(source, destination, chunkSize, 0, null, 1000); + } + + public static bool Copy(Stream source, Stream destination, int chunkSize, long totalSize, + Func progress, int progressUpdateRate) + { + byte[] buffer = new byte[chunkSize]; + int bytesRead = 0; + long totalRead = 0; + float averageSpeed = -1f; + float lastSpeed = 0f; + float smoothing = 0.005f; + long readLastSecond = 0; + long timeToFinish = 0; + Stopwatch watch = null; + bool success = true; + + bool trackProgress = totalSize > 0 && progress != null; + if (trackProgress) + watch = new Stopwatch(); + + do + { + if (trackProgress) + watch.Start(); + + bytesRead = source.Read(buffer, 0, chunkSize); + + if (trackProgress) + watch.Stop(); + + totalRead += bytesRead; + + if (bytesRead > 0) + { + destination.Write(buffer, 0, bytesRead); + if (trackProgress) + { + readLastSecond += bytesRead; + if (watch.ElapsedMilliseconds >= progressUpdateRate || totalRead == totalSize) + { + watch.Reset(); + lastSpeed = readLastSecond; + readLastSecond = 0; + averageSpeed = averageSpeed < 0f + ? lastSpeed + : smoothing * lastSpeed + (1f - smoothing) * averageSpeed; + timeToFinish = Math.Max(1L, + (long)((totalSize - totalRead) / (averageSpeed / progressUpdateRate))); + + if (!progress(totalRead, timeToFinish)) + break; + } + } + } + } while (bytesRead > 0); + + if (totalRead > 0) + destination.Flush(); + + return success; + } + } + + public static class WebRequestExtensions + { + public static WebResponse GetResponseWithoutException(this WebRequest request) + { + try + { + return request.GetResponse(); + } + catch (WebException e) + { + if (e.Response != null) + { + return e.Response; + } + + throw e; + } + } + } + + class DownloadTask : TaskBase + { + private readonly IFileSystem fileSystem; + private long bytes; + private bool restarted; + + public float Progress { get; set; } + + public DownloadTask(CancellationToken token, IFileSystem fileSystem, string url, string destination, string validationHash = null, int retryCount = 0) + : base(token) + { + this.fileSystem = fileSystem; + ValidationHash = validationHash; + RetryCount = retryCount; + Url = url; + Destination = destination; + Name = "DownloadTask"; + } + + protected override void Run(bool success) + { + base.Run(success); + + RaiseOnStart(); + + var attempts = 0; + try + { + bool result; + do + { + Logger.Trace($"Download of {Url} Attempt {attempts + 1} of {RetryCount + 1}"); + result = Download(); + if (result && ValidationHash != null) + { + var md5 = fileSystem.CalculateMD5(Destination); + result = md5.Equals(ValidationHash, StringComparison.CurrentCultureIgnoreCase); + + if (!result) + { + Logger.Warning($"Downloaded MD5 {md5} does not match {ValidationHash}. Deleting {Destination}."); + fileSystem.FileDelete(Destination); + } + else + { + Logger.Trace($"Download confirmed {md5}"); + break; + } + } + } while (attempts++ < RetryCount); + + if (!result) + { + throw new DownloadException("Error downloading file"); + } + } + catch (Exception ex) + { + Errors = ex.Message; + if (!RaiseFaultHandlers(new DownloadException("Error downloading file", ex))) + throw; + } + finally + { + RaiseOnEnd(); + } + } + + protected virtual void UpdateProgress(float progress) + { + Progress = progress; + } + + public bool Download() + { + var fileInfo = new FileInfo(Destination); + if (fileSystem.FileExists(Destination)) + { + var fileLength = fileSystem.FileLength(Destination); + if (fileLength > 0) + { + bytes = fileInfo.Length; + restarted = true; + } + else if (fileLength == 0) + { + fileSystem.FileDelete(Destination); + } + } + + var expectingResume = restarted && bytes > 0; + + var webRequest = (HttpWebRequest)WebRequest.Create(Url); + + if (expectingResume) + { + // TODO: fix classlibs to take long overloads + webRequest.AddRange((int)bytes); + } + + webRequest.Method = "GET"; + webRequest.Timeout = 3000; + + if (expectingResume) + Logger.Trace($"Resuming download of {Url} to {Destination}"); + else + Logger.Trace($"Downloading {Url} to {Destination}"); + + using (var webResponse = (HttpWebResponse) webRequest.GetResponseWithoutException()) + { + var httpStatusCode = webResponse.StatusCode; + Logger.Trace($"Downloading {Url} StatusCode:{(int)webResponse.StatusCode}"); + + if (expectingResume && httpStatusCode == HttpStatusCode.RequestedRangeNotSatisfiable) + { + UpdateProgress(1); + return true; + } + + if (!(httpStatusCode == HttpStatusCode.OK || httpStatusCode == HttpStatusCode.PartialContent)) + { + return false; + } + + var responseLength = webResponse.ContentLength; + if (expectingResume) + { + UpdateProgress(bytes / (float)responseLength); + } + + using (var responseStream = webResponse.GetResponseStream()) + { + using (var destinationStream = fileSystem.OpenWrite(Destination, FileMode.Append)) + { + if (Token.IsCancellationRequested) + return false; + + return Utils.Copy(responseStream, destinationStream, 8192, responseLength, null, 100); + } + } + } + } + + protected string Url { get; } + + protected string Destination { get; } + + public string ValidationHash { get; set; } + + protected int RetryCount { get; } + } + + class DownloadException : Exception + { + public DownloadException(string message) : base(message) + { } + + public DownloadException(string message, Exception innerException) : base(message, innerException) + { } + } + + class DownloadTextTask : TaskBase + { + public float Progress { get; set; } + + public DownloadTextTask(CancellationToken token, string url) + : base(token) + { + Url = url; + Name = "DownloadTask"; + } + + protected override string RunWithReturn(bool success) + { + var result = base.RunWithReturn(success); + + RaiseOnStart(); + + try + { + Logger.Trace($"Downloading {Url}"); + var webRequest = WebRequest.Create(Url); + webRequest.Method = "GET"; + webRequest.Timeout = 3000; + + using (var webResponse = (HttpWebResponse)webRequest.GetResponseWithoutException()) + { + var webResponseCharacterSet = webResponse.CharacterSet ?? Encoding.UTF8.BodyName; + var encoding = Encoding.GetEncoding(webResponseCharacterSet); + + using (var responseStream = webResponse.GetResponseStream()) + using (var reader = new StreamReader(responseStream, encoding)) + { + result = reader.ReadToEnd(); + } + } + } + catch (Exception ex) + { + Errors = ex.Message; + if (!RaiseFaultHandlers(new DownloadException("Error downloading text", ex))) + throw; + } + finally + { + RaiseOnEnd(result); + } + + return result; + } + + protected virtual void UpdateProgress(float progress) + { + Progress = progress; + } + + protected string Url { get; } + } +} diff --git a/src/tests/IntegrationTests/Download/DownloadTaskTests.cs b/src/tests/IntegrationTests/Download/DownloadTaskTests.cs new file mode 100644 index 000000000..8d233b69a --- /dev/null +++ b/src/tests/IntegrationTests/Download/DownloadTaskTests.cs @@ -0,0 +1,167 @@ +using System; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using GitHub.Unity; +using NUnit.Framework; + +namespace IntegrationTests.Download +{ + [TestFixture] + class DownloadTaskTests: BaseTaskManagerTest + { + private const string TestDownload = "http://ipv4.download.thinkbroadband.com/5MB.zip"; + private const string TestDownloadMD5 = "b3215c06647bc550406a9c8ccc378756"; + + [Test] + public async Task TestDownloadTask() + { + InitializeTaskManager(); + + var fileSystem = new FileSystem(); + + var downloadPath = TestBasePath.Combine("5MB.zip"); + var downloadHalfPath = TestBasePath.Combine("5MB-split.zip"); + + var downloadTask = new DownloadTask(CancellationToken.None, fileSystem, TestDownload, downloadPath); + await downloadTask.StartAwait(); + + var downloadPathBytes = fileSystem.ReadAllBytes(downloadPath); + Logger.Trace("File size {0} bytes", downloadPathBytes.Length); + + var md5Sum = fileSystem.CalculateMD5(downloadPath); + md5Sum.Should().Be(TestDownloadMD5.ToUpperInvariant()); + + var random = new Random(); + var takeCount = random.Next(downloadPathBytes.Length); + + Logger.Trace("Cutting the first {0} Bytes", downloadPathBytes.Length - takeCount); + + var cutDownloadPathBytes = downloadPathBytes.Take(takeCount).ToArray(); + fileSystem.WriteAllBytes(downloadHalfPath, cutDownloadPathBytes); + + downloadTask = new DownloadTask(CancellationToken.None, fileSystem, TestDownload, downloadHalfPath, TestDownloadMD5, 1); + await downloadTask.StartAwait(); + + var downloadHalfPathBytes = fileSystem.ReadAllBytes(downloadHalfPath); + Logger.Trace("File size {0} Bytes", downloadHalfPathBytes.Length); + + md5Sum = fileSystem.CalculateMD5(downloadPath); + md5Sum.Should().Be(TestDownloadMD5.ToUpperInvariant()); + } + + [Test] + public void TestDownloadFailure() + { + InitializeTaskManager(); + + var fileSystem = new FileSystem(); + + var downloadPath = TestBasePath.Combine("5MB.zip"); + + var taskFailed = false; + Exception exceptionThrown = null; + + var autoResetEvent = new AutoResetEvent(false); + + var downloadTask = new DownloadTask(CancellationToken.None, fileSystem, "http://www.unknown.com/5MB.gz", downloadPath, null, 1) + .Finally((b, exception) => { + taskFailed = !b; + exceptionThrown = exception; + autoResetEvent.Set(); + }); + + downloadTask.Start(); + + autoResetEvent.WaitOne(); + + taskFailed.Should().BeTrue(); + exceptionThrown.Should().NotBeNull(); + } + + [Test] + public void TestDownloadTextTask() + { + InitializeTaskManager(); + + var downloadTask = new DownloadTextTask(CancellationToken.None, "https://github.com/robots.txt"); + var result = downloadTask.Start().Result; + var resultLines = result.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + resultLines[0].Should().Be("# If you would like to crawl GitHub contact us at support@github.com."); + } + + [Test] + public void TestDownloadTextFailture() + { + InitializeTaskManager(); + + var downloadTask = new DownloadTextTask(CancellationToken.None, "https://ggggithub.com/robots.txt"); + var exceptionThrown = false; + + try + { + var result = downloadTask.Start().Result; + } + catch (Exception e) + { + exceptionThrown = true; + } + + exceptionThrown.Should().BeTrue(); + } + + [Test] + public void TestDownloadFileAndHash() + { + InitializeTaskManager(); + + var gitArchivePath = TestBasePath.Combine("git.zip"); + var gitLfsArchivePath = TestBasePath.Combine("git-lfs.zip"); + + var fileSystem = new FileSystem(); + + var downloadGitMd5Task = new DownloadTextTask(CancellationToken.None, + "https://ghfvs-installer.github.com/unity/portable_git/git.zip.MD5.txt?cb=1"); + + var downloadGitTask = new DownloadTask(CancellationToken.None, fileSystem, + "https://ghfvs-installer.github.com/unity/portable_git/git.zip", gitArchivePath, retryCount: 1); + + var downloadGitLfsMd5Task = new DownloadTextTask(CancellationToken.None, + "https://ghfvs-installer.github.com/unity/portable_git/git-lfs.zip.MD5.txt?cb=1"); + + var downloadGitLfsTask = new DownloadTask(CancellationToken.None, fileSystem, + "https://ghfvs-installer.github.com/unity/portable_git/git-lfs.zip", gitLfsArchivePath, retryCount: 1); + + var result = true; + Exception exception = null; + + var autoResetEvent = new AutoResetEvent(false); + + downloadGitMd5Task + .Then((b, s) => + { + downloadGitTask.ValidationHash = s; + }) + .Then(downloadGitTask) + .Then(downloadGitLfsMd5Task) + .Then((b, s) => + { + downloadGitLfsTask.ValidationHash = s; + }) + .Then(downloadGitLfsTask) + .Finally((b, ex) => { + result = b; + exception = ex; + autoResetEvent.Set(); + }) + .Start(); + + autoResetEvent.WaitOne(); + + result.Should().BeTrue(); + exception.Should().BeNull(); + } + } +} diff --git a/src/tests/IntegrationTests/IntegrationTests.csproj b/src/tests/IntegrationTests/IntegrationTests.csproj index 4c8d3da01..3b5b8cb08 100644 --- a/src/tests/IntegrationTests/IntegrationTests.csproj +++ b/src/tests/IntegrationTests/IntegrationTests.csproj @@ -74,6 +74,7 @@ +