diff --git a/GVFS.sln b/GVFS.sln index 1a8794c4d9..c894312f81 100644 --- a/GVFS.sln +++ b/GVFS.sln @@ -22,13 +22,16 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "GVFS", "GVFS", "{2EF2EC94-3 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.GVFlt", "GVFS\GVFS.GVFlt\GVFS.GVFlt.csproj", "{1118B427-7063-422F-83B9-5023C8EC5A7A}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GvFlt", "GVFS\GVFS.GvFltWrapper\GvFlt.vcxproj", "{FB0831AE-9997-401B-B31F-3A065FDBEB20}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GvLib", "GVFS\GVFS.GvFltWrapper\GvLib.vcxproj", "{FB0831AE-9997-401B-B31F-3A065FDBEB20}" ProjectSection(ProjectDependencies) = postProject {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258} {374BF1E5-0B2D-4D4A-BD5E-4212299DEF09} = {374BF1E5-0B2D-4D4A-BD5E-4212299DEF09} EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Common", "GVFS\GVFS.Common\GVFS.Common.csproj", "{374BF1E5-0B2D-4D4A-BD5E-4212299DEF09}" + ProjectSection(ProjectDependencies) = postProject + {A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574} + EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS", "GVFS\GVFS\GVFS.csproj", "{32220664-594C-4425-B9A0-88E0BE2F3D2A}" ProjectSection(ProjectDependencies) = postProject @@ -59,6 +62,9 @@ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GVFS.NativeTests", "GVFS\GVFS.NativeTests\GVFS.NativeTests.vcxproj", "{3771C555-B5C1-45E2-B8B7-2CEF1619CDC5}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Hooks", "GVFS\GVFS.Hooks\GVFS.Hooks.csproj", "{BDA91EE5-C684-4FC5-A90A-B7D677421917}" + ProjectSection(ProjectDependencies) = postProject + {A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574} + EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Service", "GVFS\GVFS.Service\GVFS.Service.csproj", "{B8C1DFBA-CAFD-4F7E-A1A3-E11907B5467B}" ProjectSection(ProjectDependencies) = postProject @@ -73,6 +79,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Mount", "GVFS\GVFS.Mou EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GVFS.ReadObjectHook", "GVFS\GVFS.ReadObjectHook\GVFS.ReadObjectHook.vcxproj", "{5A6656D5-81C7-472C-9DC8-32D071CB2258}" + ProjectSection(ProjectDependencies) = postProject + {A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574} + EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{28674A4B-1223-4633-A460-C8CC39B09318}" ProjectSection(SolutionItems) = preProject @@ -83,9 +92,18 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{2867 Scripts\RunUnitTests.bat = Scripts\RunUnitTests.bat EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.PerfProfiling", "GVFS\GVFS.PerfProfiling\GVFS.PerfProfiling.csproj", "{C5D3CA26-562F-4CA4-A378-B93E97A730E3}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Service.UI", "GVFS\GVFS.Service.UI\GVFS.Service.UI.csproj", "{93B403FD-DAFB-46C5-9636-B122792A548A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.PreBuild", "GVFS\GVFS.Build\GVFS.PreBuild.csproj", "{A4984251-840E-4622-AD0C-66DFCE2B2574}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{AB0D9230-3893-4486-8899-F9C871FB5D5F}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GitHooksLoader", "GitHooksLoader\GitHooksLoader.vcxproj", "{798DE293-6EDA-4DC4-9395-BE7A71C563E3}" + ProjectSection(ProjectDependencies) = postProject + {A4984251-840E-4622-AD0C-66DFCE2B2574} = {A4984251-840E-4622-AD0C-66DFCE2B2574} + EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -145,10 +163,18 @@ Global {5A6656D5-81C7-472C-9DC8-32D071CB2258}.Debug|x64.Build.0 = Debug|x64 {5A6656D5-81C7-472C-9DC8-32D071CB2258}.Release|x64.ActiveCfg = Release|x64 {5A6656D5-81C7-472C-9DC8-32D071CB2258}.Release|x64.Build.0 = Release|x64 + {C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Debug|x64.ActiveCfg = Debug|x64 + {C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Debug|x64.Build.0 = Debug|x64 + {C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Release|x64.ActiveCfg = Release|x64 + {C5D3CA26-562F-4CA4-A378-B93E97A730E3}.Release|x64.Build.0 = Release|x64 {93B403FD-DAFB-46C5-9636-B122792A548A}.Debug|x64.ActiveCfg = Debug|x64 {93B403FD-DAFB-46C5-9636-B122792A548A}.Debug|x64.Build.0 = Debug|x64 {93B403FD-DAFB-46C5-9636-B122792A548A}.Release|x64.ActiveCfg = Release|x64 {93B403FD-DAFB-46C5-9636-B122792A548A}.Release|x64.Build.0 = Release|x64 + {A4984251-840E-4622-AD0C-66DFCE2B2574}.Debug|x64.ActiveCfg = Debug|x64 + {A4984251-840E-4622-AD0C-66DFCE2B2574}.Debug|x64.Build.0 = Debug|x64 + {A4984251-840E-4622-AD0C-66DFCE2B2574}.Release|x64.ActiveCfg = Release|x64 + {A4984251-840E-4622-AD0C-66DFCE2B2574}.Release|x64.Build.0 = Release|x64 {798DE293-6EDA-4DC4-9395-BE7A71C563E3}.Debug|x64.ActiveCfg = Debug|x64 {798DE293-6EDA-4DC4-9395-BE7A71C563E3}.Debug|x64.Build.0 = Debug|x64 {798DE293-6EDA-4DC4-9395-BE7A71C563E3}.Release|x64.ActiveCfg = Release|x64 @@ -172,6 +198,8 @@ Global {17498502-AEFF-4E70-90CC-1D0B56A8ADF5} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} {28674A4B-1223-4633-A460-C8CC39B09318} = {DCE11095-DA5F-4878-B58D-2702765560F5} + {C5D3CA26-562F-4CA4-A378-B93E97A730E3} = {C41F10F9-1163-4CFA-A465-EA728F75E9FA} {93B403FD-DAFB-46C5-9636-B122792A548A} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} + {A4984251-840E-4622-AD0C-66DFCE2B2574} = {AB0D9230-3893-4486-8899-F9C871FB5D5F} EndGlobalSection EndGlobal diff --git a/GVFS/FastFetch/CheckoutFetchHelper.cs b/GVFS/FastFetch/CheckoutFetchHelper.cs index c354c181a9..9a2d5a4af8 100644 --- a/GVFS/FastFetch/CheckoutFetchHelper.cs +++ b/GVFS/FastFetch/CheckoutFetchHelper.cs @@ -66,7 +66,7 @@ public override void FastFetch(string branchOrCommit, bool isBranch) // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs - CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment); + CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); diff --git a/GVFS/FastFetch/FastFetch.csproj b/GVFS/FastFetch/FastFetch.csproj index f1966e5856..c4bf791b77 100644 --- a/GVFS/FastFetch/FastFetch.csproj +++ b/GVFS/FastFetch/FastFetch.csproj @@ -43,8 +43,9 @@ true - - ..\..\..\packages\CommandLineParser.2.0.275-beta\lib\net45\CommandLine.dll + + False + ..\..\..\packages\CommandLineParser.2.1.1-beta\lib\net45\CommandLine.dll True @@ -69,6 +70,8 @@ + + diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index c32358f31a..666fd1406f 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -5,7 +5,6 @@ using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; -using System.Diagnostics; namespace FastFetch { @@ -103,15 +102,15 @@ public class FastFetchVerb "folders", Required = false, Default = "", - HelpText = "A semicolon-delimited list of paths to fetch")] - public string PathWhitelist { get; set; } + HelpText = "A semicolon-delimited list of folders to fetch")] + public string FolderList { get; set; } [Option( "folders-list", Required = false, Default = "", - HelpText = "A file containing line-delimited list of paths to fetch")] - public string PathWhitelistFile { get; set; } + HelpText = "A file containing line-delimited list of folders to fetch")] + public string FolderListFile { get; set; } [Option( "Allow-index-metadata-update-from-working-tree", @@ -192,7 +191,7 @@ private int ExecuteWithExitCode() Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID."); } - using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch")) + using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", useCriticalTelemetryFlag: false)) { if (this.Verbose) { @@ -206,29 +205,26 @@ private int ExecuteWithExitCode() string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch"); tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any); - RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); - - string error; - CacheServerInfo cacheServer; - if (!CacheServerInfo.TryDetermineCacheServer(this.CacheServerUrl, tracer, enlistment, retryConfig, out cacheServer, out error)) - { - tracer.RelatedError(error); - return ExitFailure; - } + CacheServerInfo cacheServer = new CacheServerInfo(this.GetRemoteUrl(enlistment), null); tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, cacheServer.Url, + enlistment.GitObjectsRoot, new EventMetadata { { "TargetCommitish", commitish }, { "Checkout", this.Checkout }, }); + RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); FetchHelper fetchHelper = this.GetFetchHelper(tracer, enlistment, cacheServer, retryConfig); - if (!FetchHelper.TryLoadPathWhitelist(tracer, this.PathWhitelist, this.PathWhitelistFile, enlistment, fetchHelper.PathWhitelist)) + string error; + if (!FetchHelper.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, fetchHelper.FolderList, out error)) { + tracer.RelatedError(error); + Console.WriteLine(error); return ExitFailure; } @@ -291,7 +287,23 @@ private int ExecuteWithExitCode() return isSuccess ? ExitSuccess : ExitFailure; } } - + + private string GetRemoteUrl(Enlistment enlistment) + { + if (!string.IsNullOrWhiteSpace(this.CacheServerUrl)) + { + return this.CacheServerUrl; + } + + string configuredUrl = CacheServerResolver.GetUrlFromConfig(enlistment); + if (!string.IsNullOrWhiteSpace(configuredUrl)) + { + return configuredUrl; + } + + return enlistment.RepoUrl; + } + private FetchHelper GetFetchHelper(ITracer tracer, Enlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig) { GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig); diff --git a/GVFS/FastFetch/FetchHelper.cs b/GVFS/FastFetch/FetchHelper.cs index 61e2b09c16..d7d6eecd85 100644 --- a/GVFS/FastFetch/FetchHelper.cs +++ b/GVFS/FastFetch/FetchHelper.cs @@ -1,4 +1,5 @@ -using FastFetch.Jobs; +using FastFetch.Git; +using FastFetch.Jobs; using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.Http; @@ -47,8 +48,10 @@ public class FetchHelper this.Tracer = tracer; this.Enlistment = enlistment; this.ObjectRequestor = objectRequestor; - this.GitObjects = new GitObjects(tracer, enlistment, this.ObjectRequestor); - this.PathWhitelist = new List(); + + this.GitObjects = new FastFetchGitObjects(tracer, enlistment, this.ObjectRequestor); + this.FileList = new List(); + this.FolderList = new List(); // We never want to update config settings for a GVFSEnlistment this.SkipConfigUpdate = enlistment is GVFSEnlistment; @@ -56,41 +59,89 @@ public class FetchHelper public bool HasFailures { get; protected set; } - public List PathWhitelist { get; private set; } + public List FileList { get; } - public static bool TryLoadPathWhitelist(ITracer tracer, string pathWhitelistInput, string pathWhitelistFile, Enlistment enlistment, List pathWhitelistOutput) - { - Func makePathAbsolute = path => Path.Combine(enlistment.EnlistmentRoot, path.Replace('/', '\\').TrimStart('\\')); + public List FolderList { get; } - pathWhitelistOutput.AddRange(pathWhitelistInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(makePathAbsolute)); + public static bool TryLoadFolderList(Enlistment enlistment, string foldersInput, string folderListFile, List folderListOutput, out string error) + { + folderListOutput.AddRange( + foldersInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries) + .Select(path => FetchHelper.ToAbsolutePath(enlistment, path, isFolder: true))); - if (!string.IsNullOrWhiteSpace(pathWhitelistFile)) + if (!string.IsNullOrWhiteSpace(folderListFile)) { - if (File.Exists(pathWhitelistFile)) + if (File.Exists(folderListFile)) { - IEnumerable allLines = File.ReadAllLines(pathWhitelistFile) + IEnumerable allLines = File.ReadAllLines(folderListFile) .Select(line => line.Trim()) .Where(line => !string.IsNullOrEmpty(line)) .Where(line => !line.StartsWith(GVFSConstants.GitCommentSign.ToString())) - .Select(makePathAbsolute); + .Select(path => FetchHelper.ToAbsolutePath(enlistment, path, isFolder: true)); - pathWhitelistOutput.AddRange(allLines); + folderListOutput.AddRange(allLines); } else { - tracer.RelatedError("Could not find '{0}' for folder filtering.", pathWhitelistFile); - Console.WriteLine("Could not find '{0}' for folder filtering.", pathWhitelistFile); + error = string.Format("Could not find '{0}' for folder list.", folderListFile); + return false; + } + } + + folderListOutput.RemoveAll(string.IsNullOrWhiteSpace); + + foreach (string folder in folderListOutput) + { + if (folder.Contains("*")) + { + error = "Wildcards are not supported for folders. Invalid entry: " + folder; return false; } } - pathWhitelistOutput.RemoveAll(string.IsNullOrWhiteSpace); + error = null; + return true; + } + + public static bool TryLoadFileList(Enlistment enlistment, string filesInput, List fileListOutput, out string error) + { + fileListOutput.AddRange( + filesInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries) + .Select(path => FetchHelper.ToAbsolutePath(enlistment, path, isFolder: false))); + + foreach (string file in fileListOutput) + { + if (file.IndexOf('*', 1) != -1) + { + error = "Only prefix wildcards are supported. Invalid entry: " + file; + return false; + } + + if (file.EndsWith(GVFSConstants.GitPathSeparatorString) || + file.EndsWith(GVFSConstants.PathSeparatorString)) + { + error = "Folders are not allowed in the file list. Invalid entry: " + file; + return false; + } + } + + error = null; return true; } /// A specific branch to filter for, or null for all branches returned from info/refs public virtual void FastFetch(string branchOrCommit, bool isBranch) { + int matchedBlobs; + int downloadedBlobs; + this.FastFetchWithStats(branchOrCommit, isBranch, out matchedBlobs, out downloadedBlobs); + } + + public void FastFetchWithStats(string branchOrCommit, bool isBranch, out int matchedBlobs, out int downloadedBlobs) + { + matchedBlobs = 0; + downloadedBlobs = 0; + if (string.IsNullOrWhiteSpace(branchOrCommit)) { throw new FetchException("Must specify branch or commit to fetch"); @@ -129,7 +180,7 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch) string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check - DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.PathWhitelist); + DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); @@ -164,6 +215,9 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch) packIndexer.WaitForCompletion(); this.HasFailures |= packIndexer.HasFailures; + matchedBlobs = blobFinder.AvailableBlobCount + blobFinder.MissingBlobCount; + downloadedBlobs = blobFinder.MissingBlobCount; + if (!this.SkipConfigUpdate && !this.HasFailures) { this.UpdateRefs(branchOrCommit, isBranch, refs); @@ -263,15 +317,15 @@ protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects) using (ITracer activity = this.Tracer.StartActivity("DownloadTrees", EventLevel.Informational, Keywords.Telemetry, startMetadata)) { - using (LibGit2Repo repo = new LibGit2Repo(this.Tracer, this.Enlistment.WorkingDirectoryRoot)) + using (FastFetchLibGit2Repo repo = new FastFetchLibGit2Repo(this.Tracer, this.Enlistment.WorkingDirectoryRoot)) { if (!repo.ObjectExists(commitSha)) { - if (!gitObjects.TryDownloadAndSaveCommit(commitSha, commitDepth: CommitDepth)) + if (!gitObjects.TryEnsureCommitIsLocal(commitSha, commitDepth: CommitDepth)) { EventMetadata metadata = new EventMetadata(); metadata.Add("ObjectsEndpointUrl", this.ObjectRequestor.CacheServer.ObjectsEndpointUrl); - activity.RelatedError(metadata); + activity.RelatedError(metadata, "Could not download commits"); throw new FetchException("Could not download commits from {0}", this.ObjectRequestor.CacheServer.ObjectsEndpointUrl); } } @@ -279,6 +333,22 @@ protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects) } } + private static string ToAbsolutePath(Enlistment enlistment, string path, bool isFolder) + { + string absolute = + path.StartsWith("*") + ? path + : Path.Combine(enlistment.WorkingDirectoryRoot, path.Replace(GVFSConstants.GitPathSeparator, GVFSConstants.PathSeparator).TrimStart(GVFSConstants.PathSeparator)); + + if (isFolder && + !absolute.EndsWith(GVFSConstants.PathSeparatorString)) + { + absolute += GVFSConstants.PathSeparatorString; + } + + return absolute; + } + private bool IsSymbolicRef(string targetCommitish) { return targetCommitish.StartsWith("refs/", StringComparison.OrdinalIgnoreCase); diff --git a/GVFS/FastFetch/Git/DiffHelper.cs b/GVFS/FastFetch/Git/DiffHelper.cs index 6b123bf6f3..93f1ca8529 100644 --- a/GVFS/FastFetch/Git/DiffHelper.cs +++ b/GVFS/FastFetch/Git/DiffHelper.cs @@ -1,4 +1,6 @@ -using GVFS.Common.Tracing; +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; using System.Collections.Concurrent; @@ -6,14 +8,15 @@ using System.IO; using System.Linq; -namespace GVFS.Common.Git +namespace FastFetch.Git { public class DiffHelper { private const string AreaPath = nameof(DiffHelper); private ITracer tracer; - private List pathWhitelist; + private List fileList; + private List folderList; private HashSet filesAdded = new HashSet(StringComparer.OrdinalIgnoreCase); private HashSet stagedDirectoryOperations = new HashSet(new DiffTreeByNameComparer()); @@ -22,15 +25,16 @@ public class DiffHelper private Enlistment enlistment; private GitProcess git; - public DiffHelper(ITracer tracer, Enlistment enlistment, IEnumerable pathWhitelist) - : this(tracer, enlistment, new GitProcess(enlistment), pathWhitelist) + public DiffHelper(ITracer tracer, Enlistment enlistment, IEnumerable fileList, IEnumerable folderList) + : this(tracer, enlistment, new GitProcess(enlistment), fileList, folderList) { } - public DiffHelper(ITracer tracer, Enlistment enlistment, GitProcess git, IEnumerable pathWhitelist) + public DiffHelper(ITracer tracer, Enlistment enlistment, GitProcess git, IEnumerable fileList, IEnumerable folderList) { this.tracer = tracer; - this.pathWhitelist = new List(pathWhitelist); + this.fileList = new List(fileList); + this.folderList = new List(folderList); this.enlistment = enlistment; this.git = git; @@ -75,7 +79,7 @@ public void PerformDiff(string targetCommitSha) { string targetTreeSha; string headTreeSha; - using (LibGit2Repo repo = new LibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryRoot)) + using (FastFetchLibGit2Repo repo = new FastFetchLibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryRoot)) { targetTreeSha = repo.GetTreeSha(targetCommitSha); headTreeSha = repo.GetTreeSha("HEAD"); @@ -118,7 +122,7 @@ public void PerformDiff(string sourceTreeSha, string targetTreeSha) GitProcess.Result result = this.git.DiffTree( sourceTreeSha, targetTreeSha, - line => this.EnqueueOperationsFromDiffTreeLine(this.tracer, this.enlistment.EnlistmentRoot, line)); + line => this.EnqueueOperationsFromDiffTreeLine(this.tracer, this.enlistment.WorkingDirectoryRoot, line)); if (result.HasErrors) { @@ -196,13 +200,13 @@ private void FlushStagedQueues() private void EnqueueOperationsFromLsTreeLine(ITracer activity, string line) { - DiffTreeResult result = DiffTreeResult.ParseFromLsTreeLine(line, this.enlistment.EnlistmentRoot); + DiffTreeResult result = DiffTreeResult.ParseFromLsTreeLine(line, this.enlistment.WorkingDirectoryRoot); if (result == null) { this.tracer.RelatedError("Unrecognized ls-tree line: {0}", line); } - if (!this.ResultIsInWhitelist(result)) + if (!this.ShouldIncludeResult(result)) { return; } @@ -213,10 +217,10 @@ private void EnqueueOperationsFromLsTreeLine(ITracer activity, string line) { EventMetadata metadata = new EventMetadata(); metadata.Add("Filename", result.TargetFilename); - metadata.Add("Message", "File exists in tree with two different cases. Taking the last one."); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "File exists in tree with two different cases. Taking the last one."); this.tracer.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata); - // Since we match only on filename, readding is the easiest way to update the set. + // Since we match only on filename, re-adding is the easiest way to update the set. this.stagedDirectoryOperations.Remove(result); this.stagedDirectoryOperations.Add(result); } @@ -237,7 +241,7 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot } DiffTreeResult result = DiffTreeResult.ParseFromDiffTreeLine(line, repoRoot); - if (!this.ResultIsInWhitelist(result)) + if (!this.ShouldIncludeResult(result)) { return; } @@ -247,8 +251,7 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot { EventMetadata metadata = new EventMetadata(); metadata.Add("Path", result.TargetFilename); - metadata.Add("ErrorMessage", "Unexpected diff operation: " + result.Operation); - activity.RelatedError(metadata); + activity.RelatedError(metadata, "Unexpected diff operation: " + result.Operation); this.HasFailures = true; return; } @@ -263,7 +266,7 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot { EventMetadata metadata = new EventMetadata(); metadata.Add("Filename", result.TargetFilename); - metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory."); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory."); activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata); } @@ -274,7 +277,7 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot // This could happen if a directory was deleted and an existing directory was renamed to replace it, but with a different case. EventMetadata metadata = new EventMetadata(); metadata.Add("Filename", result.TargetFilename); - metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory."); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory."); activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata); // The target of RenameEdit is always akin to an Add, so replacing the delete is the safer thing to do. @@ -289,6 +292,12 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot this.EnqueueFileAddOperation(activity, result); } + if (!result.SourceIsDirectory) + { + // Handle when a file becomes a directory by deleting the file. + this.EnqueueFileDeleteOperation(activity, result.SourceFilename); + } + break; case DiffTreeResult.Operations.Add: case DiffTreeResult.Operations.Modify: @@ -297,7 +306,7 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot { EventMetadata metadata = new EventMetadata(); metadata.Add("Filename", result.TargetFilename); - metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory."); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory."); activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata); // Replace the delete with the add to make sure we don't delete a folder from under ourselves @@ -337,11 +346,32 @@ private void EnqueueOperationsFromDiffTreeLine(ITracer activity, string repoRoot } } - private bool ResultIsInWhitelist(DiffTreeResult blobAdd) + private bool ShouldIncludeResult(DiffTreeResult blobAdd) { - return blobAdd.TargetFilename == null || - this.pathWhitelist.Count == 0 || - this.pathWhitelist.Any(path => blobAdd.TargetFilename.StartsWith(path, StringComparison.OrdinalIgnoreCase)); + if (blobAdd.TargetFilename == null) + { + return true; + } + + if (this.fileList.Count == 0 && this.folderList.Count == 0) + { + return true; + } + + if (this.fileList.Any(path => + path.StartsWith("*") + ? blobAdd.TargetFilename.EndsWith(path.Substring(1), StringComparison.OrdinalIgnoreCase) + : blobAdd.TargetFilename.Equals(path, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + if (this.folderList.Any(path => blobAdd.TargetFilename.StartsWith(path, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + return false; } private void EnqueueFileDeleteOperation(ITracer activity, string targetPath) @@ -350,7 +380,7 @@ private void EnqueueFileDeleteOperation(ITracer activity, string targetPath) { EventMetadata metadata = new EventMetadata(); metadata.Add("Filename", targetPath); - metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory."); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory."); activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata); return; @@ -380,7 +410,7 @@ private void EnqueueFileAddOperation(ITracer activity, DiffTreeResult operation) { EventMetadata metadata = new EventMetadata(); metadata.Add("Filename", operation.TargetFilename); - metadata.Add("Message", "A case change was attempted. It will not be reflected in the working directory."); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "A case change was attempted. It will not be reflected in the working directory."); activity.RelatedEvent(EventLevel.Warning, "CaseConflict", metadata); } diff --git a/GVFS/FastFetch/Git/FastFetchGitObjects.cs b/GVFS/FastFetch/Git/FastFetchGitObjects.cs new file mode 100644 index 0000000000..cdc036f9d5 --- /dev/null +++ b/GVFS/FastFetch/Git/FastFetchGitObjects.cs @@ -0,0 +1,15 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Common.Http; +using GVFS.Common.Tracing; + +namespace FastFetch.Git +{ + public class FastFetchGitObjects : GitObjects + { + public FastFetchGitObjects(ITracer tracer, Enlistment enlistment, GitObjectsHttpRequestor objectRequestor, PhysicalFileSystem fileSystem = null) : base(tracer, enlistment, objectRequestor, fileSystem) + { + } + } +} diff --git a/GVFS/FastFetch/Git/FastFetchLibGit2Repo.cs b/GVFS/FastFetch/Git/FastFetchLibGit2Repo.cs new file mode 100644 index 0000000000..a4a41790a9 --- /dev/null +++ b/GVFS/FastFetch/Git/FastFetchLibGit2Repo.cs @@ -0,0 +1,122 @@ +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using Microsoft.Diagnostics.Tracing; +using Microsoft.Win32.SafeHandles; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; + +namespace FastFetch.Git +{ + public class FastFetchLibGit2Repo : LibGit2Repo + { + private const int AccessDeniedWin32Error = 5; + + public FastFetchLibGit2Repo(ITracer tracer, string repoPath) + : base(tracer, repoPath) + { + } + + public virtual bool TryCopyBlobToFile(string sha, IEnumerable destinations, out long bytesWritten) + { + IntPtr objHandle; + if (Native.RevParseSingle(out objHandle, this.RepoHandle, sha) != Native.SuccessCode) + { + bytesWritten = 0; + EventMetadata metadata = new EventMetadata(); + metadata.Add("ObjectSha", sha); + this.Tracer.RelatedError(metadata, "Couldn't find object"); + return false; + } + + try + { + // Avoid marshalling raw content by using byte* and native writes + unsafe + { + switch (Native.Object.GetType(objHandle)) + { + case Native.ObjectTypes.Blob: + byte* originalData = Native.Blob.GetRawContent(objHandle); + long originalSize = Native.Blob.GetRawSize(objHandle); + + foreach (string destination in destinations) + { + try + { + using (SafeFileHandle fileHandle = OpenForWrite(this.Tracer, destination)) + { + if (fileHandle.IsInvalid) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + byte* data = originalData; + long size = originalSize; + uint written = 0; + while (size > 0) + { + uint toWrite = size < uint.MaxValue ? (uint)size : uint.MaxValue; + if (!Native.WriteFile(fileHandle, data, toWrite, out written, IntPtr.Zero)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + size -= written; + data = data + written; + } + } + } + catch (Exception e) + { + this.Tracer.RelatedError("Exception writing {0}: {1}", destination, e); + throw; + } + } + + bytesWritten = originalSize * destinations.Count(); + break; + default: + throw new NotSupportedException("Copying object types other than blobs is not supported."); + } + } + } + finally + { + Native.Object.Free(objHandle); + } + + return true; + } + + private static SafeFileHandle OpenForWrite(ITracer tracer, string fileName) + { + SafeFileHandle handle = Native.CreateFile(fileName, FileAccess.Write, FileShare.None, IntPtr.Zero, FileMode.Create, FileAttributes.Normal, IntPtr.Zero); + if (handle.IsInvalid) + { + // If we get a access denied, try reverting the acls to defaults inherited by parent + if (Marshal.GetLastWin32Error() == AccessDeniedWin32Error) + { + tracer.RelatedEvent( + EventLevel.Warning, + "FailedOpenForWrite", + new EventMetadata + { + { TracingConstants.MessageKey.WarningMessage, "Received access denied. Attempting to delete." }, + { "FileName", fileName } + }); + + File.SetAttributes(fileName, FileAttributes.Normal); + File.Delete(fileName); + + handle = Native.CreateFile(fileName, FileAccess.Write, FileShare.None, IntPtr.Zero, FileMode.Create, FileAttributes.Normal, IntPtr.Zero); + } + } + + return handle; + } + } +} diff --git a/GVFS/FastFetch/GitEnlistment.cs b/GVFS/FastFetch/GitEnlistment.cs index fcacbb3857..cdb482d715 100644 --- a/GVFS/FastFetch/GitEnlistment.cs +++ b/GVFS/FastFetch/GitEnlistment.cs @@ -8,15 +8,20 @@ public class GitEnlistment : Enlistment { private GitEnlistment(string repoRoot, string gitBinPath) : base( - repoRoot, repoRoot, - Path.Combine(repoRoot, GVFSConstants.DotGit.Objects.Root), + repoRoot, null, - gitBinPath, + gitBinPath, gvfsHooksRoot: null) { + this.GitObjectsRoot = Path.Combine(repoRoot, GVFSConstants.DotGit.Objects.Root); + this.GitPackRoot = Path.Combine(this.GitObjectsRoot, GVFSConstants.DotGit.Objects.Pack.Name); } + public override string GitObjectsRoot { get; } + + public override string GitPackRoot { get; } + public string FastFetchLogRoot { get { return Path.Combine(this.EnlistmentRoot, GVFSConstants.DotGit.Root, ".fastfetch"); } diff --git a/GVFS/FastFetch/Jobs/CheckoutJob.cs b/GVFS/FastFetch/Jobs/CheckoutJob.cs index f08358bdec..61ec285ffb 100644 --- a/GVFS/FastFetch/Jobs/CheckoutJob.cs +++ b/GVFS/FastFetch/Jobs/CheckoutJob.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; +using System.Threading.Tasks; namespace FastFetch.Jobs { @@ -29,14 +30,22 @@ public class CheckoutJob : Job private long bytesWritten = 0; private long shasReceived = 0; - public CheckoutJob(int maxParallel, IEnumerable pathWhitelist, string targetCommitSha, ITracer tracer, Enlistment enlistment) - : base(maxParallel) + // Checkout requires synchronization between the delete/directory/add stages, so control the parallelization + private int maxParallel; + + public CheckoutJob(int maxParallel, IEnumerable folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment) + : base(1) { this.tracer = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata: null); this.enlistment = enlistment; - this.diff = new DiffHelper(tracer, enlistment, pathWhitelist); + this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList); this.targetCommitSha = targetCommitSha; this.AvailableBlobShas = new BlockingCollection(); + + // Keep track of how parallel we're expected to be later during DoWork + // Note that '1' is passed to the Job base object, forcing DoWork to be single threaded + // This allows us to control the synchronization between stages by doing the parallization ourselves + this.maxParallel = maxParallel; } public BlockingCollection RequiredBlobs @@ -61,40 +70,40 @@ protected override void DoBeforeWork() protected override void DoWork() { + // Do the delete operations first as they can't have dependencies on other work using (ITracer activity = this.tracer.StartActivity( - nameof(this.HandleAllDirectoryOperations), + nameof(this.HandleAllFileDeleteOperations), EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - this.HandleAllDirectoryOperations(); - + Parallel.For(1, this.maxParallel, (i) => { this.HandleAllFileDeleteOperations(); }); EventMetadata metadata = new EventMetadata(); - metadata.Add("DirectoryOperationsCompleted", this.directoryOpCount); + metadata.Add("FilesDeleted", this.fileDeleteCount); activity.Stop(metadata); } + // Do directory operations after deletes in case a file delete must be done first using (ITracer activity = this.tracer.StartActivity( - nameof(this.HandleAllFileDeleteOperations), + nameof(this.HandleAllDirectoryOperations), EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - this.HandleAllFileDeleteOperations(); - + Parallel.For(1, this.maxParallel, (i) => { this.HandleAllDirectoryOperations(); }); EventMetadata metadata = new EventMetadata(); - metadata.Add("FilesDeleted", this.fileDeleteCount); + metadata.Add("DirectoryOperationsCompleted", this.directoryOpCount); activity.Stop(metadata); } + // Do add operations last, after all deletes and directories have been created using (ITracer activity = this.tracer.StartActivity( nameof(this.HandleAllFileAddOperations), EventLevel.Informational, Keywords.Telemetry, metadata: null)) { - this.HandleAllFileAddOperations(); - + Parallel.For(1, this.maxParallel, (i) => { this.HandleAllFileAddOperations(); }); EventMetadata metadata = new EventMetadata(); metadata.Add("FilesWritten", this.fileWriteCount); activity.Stop(metadata); @@ -109,7 +118,6 @@ protected override void DoAfterWork() { this.HasFailures = true; EventMetadata errorMetadata = new EventMetadata(); - errorMetadata.Add("ErrorMessage", "Not all file writes were completed"); if (this.diff.FileAddOperations.Count < 10) { errorMetadata.Add("RemainingShas", string.Join(",", this.diff.FileAddOperations.Keys)); @@ -119,7 +127,7 @@ protected override void DoAfterWork() errorMetadata.Add("RemainingShaCount", this.diff.FileAddOperations.Count); } - this.tracer.RelatedError(errorMetadata); + this.tracer.RelatedError(errorMetadata, "Not all file writes were completed"); } this.AddedOrEditedLocalFiles.CompleteAdding(); @@ -156,8 +164,7 @@ private void HandleAllDirectoryOperations() EventMetadata metadata = new EventMetadata(); metadata.Add("Operation", "CreateDirectory"); metadata.Add("Path", treeOp.TargetFilename); - metadata.Add("ErrorMessage", ex.Message); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, ex.Message); this.HasFailures = true; } @@ -178,8 +185,7 @@ private void HandleAllDirectoryOperations() EventMetadata metadata = new EventMetadata(); metadata.Add("Operation", "DeleteDirectory"); metadata.Add("Path", treeOp.TargetFilename); - metadata.Add("ErrorMessage", ex.Message); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, ex.Message); this.HasFailures = true; } } @@ -203,6 +209,7 @@ private void HandleAllDirectoryOperations() { if (File.Exists(treeOp.SourceFilename)) { + File.SetAttributes(treeOp.SourceFilename, FileAttributes.Normal); File.Delete(treeOp.SourceFilename); } @@ -220,8 +227,7 @@ private void HandleAllDirectoryOperations() EventMetadata metadata = new EventMetadata(); metadata.Add("Operation", "RenameDirectory"); metadata.Add("Path", treeOp.TargetFilename); - metadata.Add("ErrorMessage", ex.Message); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, ex.Message); this.HasFailures = true; } @@ -265,8 +271,7 @@ private void HandleAllFileDeleteOperations() EventMetadata metadata = new EventMetadata(); metadata.Add("Operation", "DeleteFile"); metadata.Add("Path", path); - metadata.Add("ErrorMessage", ex.Message); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, ex.Message); this.HasFailures = true; } } @@ -274,7 +279,7 @@ private void HandleAllFileDeleteOperations() private void HandleAllFileAddOperations() { - using (LibGit2Repo repo = new LibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryRoot)) + using (FastFetchLibGit2Repo repo = new FastFetchLibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryRoot)) { string availableBlob; while (this.AvailableBlobShas.TryTake(out availableBlob, millisecondsTimeout: -1)) @@ -317,8 +322,7 @@ private void HandleAllFileAddOperations() { EventMetadata errorData = new EventMetadata(); errorData.Add("Operation", "WriteFile"); - errorData.Add("ErrorMessage", ex.ToString()); - this.tracer.RelatedError(errorData); + this.tracer.RelatedError(errorData, ex.ToString()); this.HasFailures = true; } } diff --git a/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs b/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs index 0464c9966e..acb4b13274 100644 --- a/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs +++ b/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs @@ -1,12 +1,8 @@ -using FastFetch.Jobs.Data; +using FastFetch.Git; using GVFS.Common; -using GVFS.Common.Git; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; -using System; using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; using System.Threading; namespace FastFetch.Jobs @@ -18,12 +14,12 @@ public class FindMissingBlobsJob : Job { private const string AreaPath = nameof(FindMissingBlobsJob); private const string TreeSearchAreaPath = "TreeSearch"; - + private ITracer tracer; private Enlistment enlistment; private int missingBlobCount; private int availableBlobCount; - + private BlockingCollection inputQueue; private ConcurrentHashSet alreadyFoundBlobIds; @@ -40,7 +36,7 @@ public class FindMissingBlobsJob : Job this.inputQueue = inputQueue; this.enlistment = enlistment; this.alreadyFoundBlobIds = new ConcurrentHashSet(); - + this.DownloadQueue = new BlockingCollection(); this.AvailableBlobs = availableBlobs; } @@ -48,10 +44,20 @@ public class FindMissingBlobsJob : Job public BlockingCollection DownloadQueue { get; } public BlockingCollection AvailableBlobs { get; } + public int MissingBlobCount + { + get { return this.missingBlobCount; } + } + + public int AvailableBlobCount + { + get { return this.availableBlobCount; } + } + protected override void DoWork() { string blobId; - using (LibGit2Repo repo = new LibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryRoot)) + using (FastFetchLibGit2Repo repo = new FastFetchLibGit2Repo(this.tracer, this.enlistment.WorkingDirectoryRoot)) { while (this.inputQueue.TryTake(out blobId, Timeout.Infinite)) { diff --git a/GVFS/FastFetch/Jobs/IndexPackJob.cs b/GVFS/FastFetch/Jobs/IndexPackJob.cs index f82b0feccd..c785f1b46e 100644 --- a/GVFS/FastFetch/Jobs/IndexPackJob.cs +++ b/GVFS/FastFetch/Jobs/IndexPackJob.cs @@ -49,8 +49,7 @@ protected override void DoWork() { EventMetadata errorMetadata = new EventMetadata(); errorMetadata.Add("PackId", request.DownloadRequest.PackId); - errorMetadata.Add("ErrorMessage", result.Errors); - activity.RelatedError(errorMetadata); + activity.RelatedError(errorMetadata, result.Errors); this.HasFailures = true; } diff --git a/GVFS/FastFetch/packages.config b/GVFS/FastFetch/packages.config index 01f5af5ffa..919f2a893d 100644 --- a/GVFS/FastFetch/packages.config +++ b/GVFS/FastFetch/packages.config @@ -1,6 +1,6 @@  - + diff --git a/GVFS/GVFS.Build/GVFS.PreBuild.csproj b/GVFS/GVFS.Build/GVFS.PreBuild.csproj new file mode 100644 index 0000000000..d9216ff85e --- /dev/null +++ b/GVFS/GVFS.Build/GVFS.PreBuild.csproj @@ -0,0 +1,72 @@ + + + + + Debug + AnyCPU + {A4984251-840E-4622-AD0C-66DFCE2B2574} + Library + Properties + GVFS.PreBuild + GVFS.PreBuild + v4.5.2 + 512 + + + true + ..\..\..\BuildOutput\GVFS.Build\bin\x64\Debug\ + ..\..\..\BuildOutput\GVFS.Build\obj\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + true + true + + + ..\..\..\BuildOutput\GVFS.Build\bin\x64\Release\ + ..\..\..\BuildOutput\GVFS.Build\obj\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + + + + + + + + + + + + + + + 0.2.173.2 + $(ProjectDir)..\..\..\packages + $(ProjectDir)..\..\..\BuildOutput + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS.Build/GenerateVersionInfo.cs b/GVFS/GVFS.Build/GenerateVersionInfo.cs new file mode 100644 index 0000000000..43863ff57a --- /dev/null +++ b/GVFS/GVFS.Build/GenerateVersionInfo.cs @@ -0,0 +1,45 @@ +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.IO; + +namespace GVFS.PreBuild +{ + public class GenerateVersionInfo : Task + { + [Required] + public string Version { get; set; } + + [Required] + public string BuildOutputPath { get; set; } + + public override bool Execute() + { + this.Log.LogMessage(MessageImportance.High, "Creating version files"); + + File.WriteAllText( + Path.Combine(this.BuildOutputPath, "CommonAssemblyVersion.cs"), + string.Format( +@"using System.Reflection; + +[assembly: AssemblyVersion(""{0}"")] +[assembly: AssemblyFileVersion(""{0}"")] +", + this.Version)); + + string commaDelimetedVersion = this.Version.Replace('.', ','); + File.WriteAllText( + Path.Combine(this.BuildOutputPath, "CommonVersionHeader.h"), + string.Format( +@" +#define GVFS_FILE_VERSION {0} +#define GVFS_FILE_VERSION_STRING ""{1}"" +#define GVFS_PRODUCT_VERSION {0} +#define GVFS_PRODUCT_VERSION_STRING ""{1}"" +", + commaDelimetedVersion, + this.Version)); + + return true; + } + } +} diff --git a/GVFS/GVFS.Common/Enlistment.cs b/GVFS/GVFS.Common/Enlistment.cs index 5db1e6da77..145e0bb41a 100644 --- a/GVFS/GVFS.Common/Enlistment.cs +++ b/GVFS/GVFS.Common/Enlistment.cs @@ -1,7 +1,6 @@ using GVFS.Common.Git; using System; using System.IO; -using System.Linq; namespace GVFS.Common { @@ -9,11 +8,10 @@ public abstract class Enlistment { private const string DeprecatedObjectsEndpointGitConfigName = "gvfs.objects-endpoint"; private const string CacheEndpointGitConfigSuffix = ".cache-server-url"; - + protected Enlistment( string enlistmentRoot, string workingDirectoryRoot, - string gitObjectsRoot, string repoUrl, string gitBinPath, string gvfsHooksRoot) @@ -25,12 +23,10 @@ public abstract class Enlistment this.EnlistmentRoot = enlistmentRoot; this.WorkingDirectoryRoot = workingDirectoryRoot; - this.GitObjectsRoot = gitObjectsRoot; + this.DotGitRoot = Path.Combine(this.WorkingDirectoryRoot, GVFSConstants.DotGit.Root); this.GitBinPath = gitBinPath; this.GVFSHooksRoot = gvfsHooksRoot; - this.SetComputedPaths(); - if (repoUrl != null) { this.RepoUrl = repoUrl; @@ -57,8 +53,8 @@ public abstract class Enlistment public string EnlistmentRoot { get; } public string WorkingDirectoryRoot { get; } public string DotGitRoot { get; private set; } - public string GitObjectsRoot { get; private set; } - public string GitPackRoot { get; private set; } + public abstract string GitObjectsRoot { get; } + public abstract string GitPackRoot { get; } public string RepoUrl { get; } public string GitBinPath { get; } @@ -87,11 +83,10 @@ public static string GetNewLogFileName(string logsRoot, string prefix) return fullPath; } - - private void SetComputedPaths() + + public virtual GitProcess CreateGitProcess() { - this.DotGitRoot = Path.Combine(this.WorkingDirectoryRoot, GVFSConstants.DotGit.Root); - this.GitPackRoot = Path.Combine(this.GitObjectsRoot, GVFSConstants.DotGit.Objects.Pack.Name); + return new GitProcess(this); } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/FileBasedCollection.cs b/GVFS/GVFS.Common/FileBasedCollection.cs new file mode 100644 index 0000000000..8980b1f689 --- /dev/null +++ b/GVFS/GVFS.Common/FileBasedCollection.cs @@ -0,0 +1,442 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Text; +using System.Threading; + +namespace GVFS.Common +{ + public abstract class FileBasedCollection : IDisposable + { + private const string EtwArea = nameof(FileBasedCollection); + + private const string AddEntryPrefix = "A "; + private const string RemoveEntryPrefix = "D "; + private const int IoFailureRetryDelayMS = 50; + private const int IoFailureLoggingThreshold = 500; + + /// + /// If true, this FileBasedCollection appends directly to dataFileHandle stream + /// If false, this FileBasedCollection only using .tmp + rename to update data on disk + /// + private readonly bool collectionAppendsDirectlyToFile; + + private readonly object fileLock = new object(); + + private readonly PhysicalFileSystem fileSystem; + private readonly string dataDirectoryPath; + private readonly string tempFilePath; + + private Stream dataFileHandle; + private ITracer tracer; + + protected FileBasedCollection(ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath, bool collectionAppendsDirectlyToFile) + { + this.tracer = tracer; + this.fileSystem = fileSystem; + this.DataFilePath = dataFilePath; + this.tempFilePath = this.DataFilePath + ".tmp"; + this.dataDirectoryPath = Path.GetDirectoryName(this.DataFilePath); + this.collectionAppendsDirectlyToFile = collectionAppendsDirectlyToFile; + } + + protected delegate bool TryParseAdd(string line, out TKey key, out TValue value, out string error); + protected delegate bool TryParseRemove(string line, out TKey key, out string error); + + public string DataFilePath { get; } + + public void Dispose() + { + lock (this.fileLock) + { + this.CloseDataFile(); + } + } + + protected void WriteAndReplaceDataFile(Func> getDataLines) + { + lock (this.fileLock) + { + try + { + this.CloseDataFile(); + + bool tmpFileCreated = false; + int tmpFileCreateAttempts = 0; + + bool tmpFileMoved = false; + int tmpFileMoveAttempts = 0; + + Exception lastException = null; + + while (!tmpFileCreated || !tmpFileMoved) + { + if (!tmpFileCreated) + { + tmpFileCreated = this.TryWriteTempFile(getDataLines, out lastException); + if (!tmpFileCreated) + { + if (this.tracer != null && tmpFileCreateAttempts % IoFailureLoggingThreshold == 0) + { + EventMetadata metadata = CreateEventMetadata(lastException); + metadata.Add("tmpFileCreateAttempts", tmpFileCreateAttempts); + this.tracer.RelatedWarning(metadata, nameof(this.WriteAndReplaceDataFile) + ": Failed to create tmp file ... retrying"); + } + + ++tmpFileCreateAttempts; + Thread.Sleep(IoFailureRetryDelayMS); + } + } + + if (tmpFileCreated) + { + try + { + if (this.fileSystem.FileExists(this.tempFilePath)) + { + this.fileSystem.MoveAndOverwriteFile(this.tempFilePath, this.DataFilePath); + tmpFileMoved = true; + } + else + { + if (this.tracer != null) + { + EventMetadata metadata = CreateEventMetadata(); + metadata.Add("tmpFileMoveAttempts", tmpFileMoveAttempts); + this.tracer.RelatedWarning(metadata, nameof(this.WriteAndReplaceDataFile) + ": tmp file is missing. Recreating tmp file."); + } + + tmpFileCreated = false; + } + } + catch (Win32Exception e) + { + if (this.tracer != null && tmpFileMoveAttempts % IoFailureLoggingThreshold == 0) + { + EventMetadata metadata = CreateEventMetadata(e); + metadata.Add("tmpFileMoveAttempts", tmpFileMoveAttempts); + this.tracer.RelatedWarning(metadata, nameof(this.WriteAndReplaceDataFile) + ": Failed to overwrite data file ... retrying"); + } + + ++tmpFileMoveAttempts; + Thread.Sleep(IoFailureRetryDelayMS); + } + } + } + + if (this.collectionAppendsDirectlyToFile) + { + this.OpenOrCreateDataFile(retryUntilSuccess: true); + } + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + } + + protected string FormatAddLine(string line) + { + return AddEntryPrefix + line; + } + + protected string FormatRemoveLine(string line) + { + return RemoveEntryPrefix + line; + } + + /// An optional callback to be run as soon as the fileLock is taken. + protected void WriteAddEntry(string value, Action synchronizedAction = null) + { + lock (this.fileLock) + { + string line = this.FormatAddLine(value); + if (synchronizedAction != null) + { + synchronizedAction(); + } + + this.WriteToDisk(line); + } + } + + /// An optional callback to be run as soon as the fileLock is taken. + protected void WriteRemoveEntry(string key, Action synchronizedAction = null) + { + lock (this.fileLock) + { + string line = this.FormatRemoveLine(key); + if (synchronizedAction != null) + { + synchronizedAction(); + } + + this.WriteToDisk(line); + } + } + + protected void DeleteDataFileIfCondition(Func condition) + { + if (!this.collectionAppendsDirectlyToFile) + { + throw new InvalidOperationException(nameof(this.DeleteDataFileIfCondition) + " requires that collectionAppendsDirectlyToFile be true"); + } + + lock (this.fileLock) + { + if (condition()) + { + this.dataFileHandle.SetLength(0); + } + } + } + + /// An optional callback to be run as soon as the fileLock is taken + protected bool TryLoadFromDisk( + TryParseAdd tryParseAdd, + TryParseRemove tryParseRemove, + Action add, + out string error, + Action synchronizedAction = null) + { + lock (this.fileLock) + { + try + { + if (synchronizedAction != null) + { + synchronizedAction(); + } + + this.fileSystem.CreateDirectory(this.dataDirectoryPath); + + this.OpenOrCreateDataFile(retryUntilSuccess: false); + + this.RemoveLastEntryIfInvalid(); + + long lineCount = 0; + + this.dataFileHandle.Seek(0, SeekOrigin.Begin); + StreamReader reader = new StreamReader(this.dataFileHandle); + Dictionary parsedEntries = new Dictionary(); + while (!reader.EndOfStream) + { + lineCount++; + + // StreamReader strips the trailing /r/n + string line = reader.ReadLine(); + if (line.StartsWith(RemoveEntryPrefix)) + { + TKey key; + if (!tryParseRemove(line.Substring(RemoveEntryPrefix.Length), out key, out error)) + { + error = string.Format("{0} is corrupt on line {1}: {2}", this.GetType().Name, lineCount, error); + return false; + } + + parsedEntries.Remove(key); + } + else if (line.StartsWith(AddEntryPrefix)) + { + TKey key; + TValue value; + if (!tryParseAdd(line.Substring(AddEntryPrefix.Length), out key, out value, out error)) + { + error = string.Format("{0} is corrupt on line {1}: {2}", this.GetType().Name, lineCount, error); + return false; + } + + parsedEntries[key] = value; + } + else + { + error = string.Format("{0} is corrupt on line {1}: Invalid Prefix '{2}'", this.GetType().Name, lineCount, line[0]); + return false; + } + } + + foreach (KeyValuePair kvp in parsedEntries) + { + add(kvp.Key, kvp.Value); + } + + if (!this.collectionAppendsDirectlyToFile) + { + this.CloseDataFile(); + } + } + catch (IOException ex) + { + error = ex.ToString(); + this.CloseDataFile(); + return false; + } + catch (Exception e) + { + this.CloseDataFile(); + throw new FileBasedCollectionException(e); + } + + error = null; + return true; + } + } + + private static EventMetadata CreateEventMetadata(Exception e = null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + if (e != null) + { + metadata.Add("Exception", e.ToString()); + } + + return metadata; + } + + /// + /// Closes dataFileHandle. Requires fileLock. + /// + private void CloseDataFile() + { + if (this.dataFileHandle != null) + { + this.dataFileHandle.Dispose(); + this.dataFileHandle = null; + } + } + + /// + /// Opens dataFileHandle for ReadWrite. Requires fileLock. + /// + /// If true, OpenOrCreateDataFile will continue to retry until it succeeds + /// If retryUntilSuccess is true, OpenOrCreateDataFile will only attempt to retry when the error is non-fatal + private void OpenOrCreateDataFile(bool retryUntilSuccess) + { + int attempts = 0; + Exception lastException = null; + while (true) + { + try + { + if (this.dataFileHandle == null) + { + this.dataFileHandle = this.fileSystem.OpenFileStream(this.DataFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read); + } + + this.dataFileHandle.Seek(0, SeekOrigin.End); + return; + } + catch (IOException e) + { + lastException = e; + } + catch (UnauthorizedAccessException e) + { + lastException = e; + } + + if (retryUntilSuccess) + { + if (this.tracer != null && attempts % IoFailureLoggingThreshold == 0) + { + EventMetadata metadata = CreateEventMetadata(lastException); + metadata.Add("attempts", attempts); + this.tracer.RelatedWarning(metadata, nameof(this.OpenOrCreateDataFile) + ": Failed to open data file stream ... retrying"); + } + + ++attempts; + Thread.Sleep(IoFailureRetryDelayMS); + } + else + { + throw lastException; + } + } + } + + /// + /// Writes data as UTF8 to dataFileHandle. fileLock will be acquired. + /// + private void WriteToDisk(string value) + { + if (!this.collectionAppendsDirectlyToFile) + { + throw new InvalidOperationException(nameof(this.WriteToDisk) + " requires that collectionAppendsDirectlyToFile be true"); + } + + byte[] bytes = Encoding.UTF8.GetBytes(value + "\r\n"); + lock (this.fileLock) + { + this.dataFileHandle.Write(bytes, 0, bytes.Length); + this.dataFileHandle.Flush(); + } + } + + /// + /// Reads entries from dataFileHandle, removing any data after the last \r\n. Requires fileLock. + /// + private void RemoveLastEntryIfInvalid() + { + if (this.dataFileHandle.Length > 2) + { + this.dataFileHandle.Seek(-2, SeekOrigin.End); + if (this.dataFileHandle.ReadByte() != '\r' || + this.dataFileHandle.ReadByte() != '\n') + { + this.dataFileHandle.Seek(0, SeekOrigin.Begin); + long lastLineEnding = 0; + while (this.dataFileHandle.Position < this.dataFileHandle.Length) + { + if (this.dataFileHandle.ReadByte() == '\r' && this.dataFileHandle.ReadByte() == '\n') + { + lastLineEnding = this.dataFileHandle.Position; + } + } + + this.dataFileHandle.SetLength(lastLineEnding); + } + } + } + + /// + /// Attempts to write all data lines to tmp file + /// + /// Method that returns the dataLines to write as an IEnumerable + /// Output parameter that's set when TryWriteTempFile catches a non-fatal exception + /// True if the write succeeded and false otherwise + /// If a fatal exception is encountered while trying to write the temp file, this method will not catch it. + private bool TryWriteTempFile(Func> getDataLines, out Exception handledException) + { + handledException = null; + + try + { + using (Stream tempFile = this.fileSystem.OpenFileStream(this.tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None)) + using (StreamWriter writer = new StreamWriter(tempFile)) + { + foreach (string line in getDataLines()) + { + writer.Write(line + "\r\n"); + } + } + + return true; + } + catch (IOException e) + { + handledException = e; + return false; + } + catch (UnauthorizedAccessException e) + { + handledException = e; + return false; + } + } + } +} diff --git a/GVFS/GVFS.Common/FileBasedCollectionException.cs b/GVFS/GVFS.Common/FileBasedCollectionException.cs new file mode 100644 index 0000000000..b86000cbfc --- /dev/null +++ b/GVFS/GVFS.Common/FileBasedCollectionException.cs @@ -0,0 +1,12 @@ +using System; + +namespace GVFS.Common +{ + public class FileBasedCollectionException : Exception + { + public FileBasedCollectionException(Exception innerException) + : base(innerException.Message, innerException) + { + } + } +} diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs new file mode 100644 index 0000000000..a0031cb469 --- /dev/null +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -0,0 +1,131 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using Newtonsoft.Json; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace GVFS.Common +{ + public class FileBasedDictionary : FileBasedCollection + { + private ConcurrentDictionary data = new ConcurrentDictionary(); + + private FileBasedDictionary(ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath) + : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) + { + } + + public static bool TryCreate(ITracer tracer, string dictionaryPath, PhysicalFileSystem fileSystem, out FileBasedDictionary output, out string error) + { + output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath); + if (!output.TryLoadFromDisk( + output.TryParseAddLine, + output.TryParseRemoveLine, + output.HandleAddLine, + out error)) + { + output = null; + return false; + } + + return true; + } + + public void SetValueAndFlush(TKey key, TValue value) + { + try + { + this.data[key] = value; + this.Flush(); + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + + public bool TryGetValue(TKey key, out TValue value) + { + try + { + return this.data.TryGetValue(key, out value); + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + + public void RemoveAndFlush(TKey key) + { + try + { + TValue value; + if (this.data.TryRemove(key, out value)) + { + this.Flush(); + } + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + + private void Flush() + { + this.WriteAndReplaceDataFile(this.GenerateDataLines); + } + + private bool TryParseAddLine(string line, out TKey key, out TValue value, out string error) + { + try + { + KeyValuePair kvp = JsonConvert.DeserializeObject>(line); + key = kvp.Key; + value = kvp.Value; + } + catch (JsonException ex) + { + key = default(TKey); + value = default(TValue); + error = "Could not deserialize JSON for add line: " + ex.Message; + return false; + } + + error = null; + return true; + } + + private bool TryParseRemoveLine(string line, out TKey key, out string error) + { + try + { + key = JsonConvert.DeserializeObject(line); + } + catch (JsonException ex) + { + key = default(TKey); + error = "Could not deserialize JSON for delete line: " + ex.Message; + return false; + } + + error = null; + return true; + } + + private void HandleAddLine(TKey key, TValue value) + { + this.data.TryAdd(key, value); + } + + private IEnumerable GenerateDataLines() + { + foreach (KeyValuePair kvp in this.data) + { + yield return this.FormatAddLine(JsonConvert.SerializeObject(kvp).Trim()); + } + } + } +} diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index b344c96e02..b74b36333c 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -10,8 +10,10 @@ namespace GVFS.Common { public class FileBasedLock : IDisposable { + private const int HResultErrorFileExists = -2147024816; // -2147024816 = 0x80070050 = ERROR_FILE_EXISTS private const int DefaultStreamWriterBufferSize = 1024; // Copied from: http://referencesource.microsoft.com/#mscorlib/system/io/streamwriter.cs,5516ce201dc06b5f private const long InvalidFileLength = -1; + private const string EtwArea = nameof(FileBasedLock); private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false, true); // Default encoding used by StreamWriter private readonly object deleteOnCloseStreamLock = new object(); @@ -20,24 +22,14 @@ public class FileBasedLock : IDisposable private ITracer tracer; private FileStream deleteOnCloseStream; - public FileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature, ExistingLockCleanup existingLockCleanup) + public FileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) { this.fileSystem = fileSystem; this.tracer = tracer; this.lockPath = lockPath; this.Signature = signature; - if (existingLockCleanup != ExistingLockCleanup.LeaveExisting) - { - this.CleanupStaleLock(existingLockCleanup); - } - } - - public enum ExistingLockCleanup - { - LeaveExisting, - DeleteExisting, - DeleteExistingAndLogSignature + this.CleanupStaleLock(); } public string Signature { get; private set; } @@ -56,9 +48,9 @@ public bool TryAcquireLockAndDeleteOnClose() this.deleteOnCloseStream = (FileStream)this.fileSystem.OpenFileStream( this.lockPath, FileMode.CreateNew, - (FileAccess)(NativeMethods.FileAccess.FILE_GENERIC_READ | NativeMethods.FileAccess.FILE_GENERIC_WRITE | NativeMethods.FileAccess.DELETE), - NativeMethods.FileAttributes.FILE_FLAG_DELETE_ON_CLOSE, - FileShare.Read); + FileAccess.ReadWrite, + FileShare.Read, + FileOptions.DeleteOnClose); // Pass in true for leaveOpen to ensure that lockStream stays open using (StreamWriter writer = new StreamWriter( @@ -73,31 +65,37 @@ public bool TryAcquireLockAndDeleteOnClose() return true; } } - catch (NativeMethods.Win32FileExistsException) + catch (IOException e) { + if (e.HResult != HResultErrorFileExists) + { + EventMetadata metadata = this.CreateLockMetadata(e); + this.tracer.RelatedWarning(metadata, "TryAcquireLockAndDeleteOnClose: IOException caught while trying to acquire lock"); + } + this.DisposeStream(); return false; } - catch (IOException e) + catch (UnauthorizedAccessException e) { - EventMetadata metadata = this.CreateLockMetadata("IOException caught while trying to acquire lock", e); - this.tracer.RelatedEvent(EventLevel.Warning, "TryAcquireLockAndDeleteOnClose", metadata); + EventMetadata metadata = this.CreateLockMetadata(e); + this.tracer.RelatedWarning(metadata, "TryAcquireLockAndDeleteOnClose: UnauthorizedAccessException caught while trying to acquire lock"); this.DisposeStream(); return false; } catch (Win32Exception e) { - EventMetadata metadata = this.CreateLockMetadata("Win32Exception caught while trying to acquire lock", e); - this.tracer.RelatedEvent(EventLevel.Warning, "TryAcquireLockAndDeleteOnClose", metadata); + EventMetadata metadata = this.CreateLockMetadata(e); + this.tracer.RelatedWarning(metadata, "TryAcquireLockAndDeleteOnClose: Win32Exception caught while trying to acquire lock"); this.DisposeStream(); return false; } catch (Exception e) { - EventMetadata metadata = this.CreateLockMetadata("Unhandled exception caught while trying to acquire lock", e); - this.tracer.RelatedError("TryAcquireLockAndDeleteOnClose", metadata); + EventMetadata metadata = this.CreateLockMetadata(e); + this.tracer.RelatedError(metadata, "TryAcquireLockAndDeleteOnClose: Unhandled exception caught while trying to acquire lock"); this.DisposeStream(); throw; @@ -128,8 +126,8 @@ public bool TryReleaseLock() } catch (IOException e) { - EventMetadata metadata = this.CreateLockMetadata("IOException caught while trying to release lock", e); - this.tracer.RelatedEvent(EventLevel.Warning, "TryReleaseLock", metadata); + EventMetadata metadata = this.CreateLockMetadata(e); + this.tracer.RelatedWarning(metadata, "TryReleaseLock: IOException caught while trying to release lock"); return false; } @@ -187,21 +185,13 @@ private bool LockFileExists() return this.fileSystem.FileExists(this.lockPath); } - private void CleanupStaleLock(ExistingLockCleanup existingLockCleanup) + private void CleanupStaleLock() { if (!this.LockFileExists()) { return; } - if (existingLockCleanup == ExistingLockCleanup.LeaveExisting) - { - throw new ArgumentException("CleanupStaleLock should not be called with LeaveExisting"); - } - - EventMetadata metadata = this.CreateLockMetadata(); - metadata.Add("existingLockCleanup", existingLockCleanup.ToString()); - long length = InvalidFileLength; try { @@ -210,61 +200,23 @@ private void CleanupStaleLock(ExistingLockCleanup existingLockCleanup) } catch (Exception e) { + EventMetadata metadata = this.CreateLockMetadata(); metadata.Add("Exception", "Exception while getting lock file length: " + e.ToString()); this.tracer.RelatedEvent(EventLevel.Warning, "CleanupEmptyLock", metadata); } if (length == 0) { - metadata.Add("Message", "Deleting empty lock file: " + this.lockPath); + EventMetadata metadata = this.CreateLockMetadata(); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "Deleting empty lock file: " + this.lockPath); this.tracer.RelatedEvent(EventLevel.Warning, "CleanupEmptyLock", metadata); } else { + EventMetadata metadata = this.CreateLockMetadata(); metadata.Add("Length", length == InvalidFileLength ? "Invalid" : length.ToString()); - - switch (existingLockCleanup) - { - case ExistingLockCleanup.DeleteExisting: - metadata.Add("Message", "Deleting stale lock file: " + this.lockPath); - this.tracer.RelatedEvent(EventLevel.Informational, "CleanupExistingLock", metadata); - break; - - case ExistingLockCleanup.DeleteExistingAndLogSignature: - string existingSignature; - try - { - string dummyLockerMessage; - this.ReadLockFile(out existingSignature, out dummyLockerMessage); - } - catch (Win32Exception e) - { - if (e.ErrorCode == NativeMethods.ERROR_FILE_NOT_FOUND) - { - // File was deleted before we could read its contents - return; - } - - throw; - } - - if (existingSignature == this.Signature) - { - metadata.Add("Message", "Deleting stale lock file: " + this.lockPath); - this.tracer.RelatedEvent(EventLevel.Informational, "CleanupExistingLock", metadata); - } - else - { - metadata.Add("ExistingLockSignature", existingSignature); - metadata.Add("Message", "Deleting stale lock file: " + this.lockPath + " with mismatched signature"); - this.tracer.RelatedEvent(EventLevel.Warning, "CleanupSignatureMismatchLock", metadata); - } - - break; - - default: - throw new InvalidOperationException("Invalid ExistingLockCleanup"); - } + metadata.Add(TracingConstants.MessageKey.InfoMessage, "Deleting stale lock file: " + this.lockPath); + this.tracer.RelatedEvent(EventLevel.Informational, "CleanupExistingLock", metadata); } this.fileSystem.DeleteFile(this.lockPath); @@ -279,18 +231,19 @@ private void WriteSignatureAndMessage(StreamWriter writer, string message) } } - private EventMetadata CreateLockMetadata(string message = null, Exception exception = null, bool errorMessage = false) + private EventMetadata CreateLockMetadata() { EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", "FileBasedLock"); + metadata.Add("Area", EtwArea); metadata.Add("LockPath", this.lockPath); metadata.Add("Signature", this.Signature); - if (message != null) - { - metadata.Add(errorMessage ? "ErrorMessage" : "Message", message); - } + return metadata; + } + private EventMetadata CreateLockMetadata(Exception exception = null) + { + EventMetadata metadata = this.CreateLockMetadata(); if (exception != null) { metadata.Add("Exception", exception.ToString()); diff --git a/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs b/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs index 8df45d8ae5..d879652d6c 100644 --- a/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs +++ b/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs @@ -15,8 +15,6 @@ public class GvFltFilter public const string GvFltTimeoutValue = "CommandTimeoutInMs"; private const string EtwArea = nameof(GvFltFilter); - private const int MinGvFltTimeoutMs = 86400000; - private const string GvFltName = "gvflt"; private const uint OkResult = 0; @@ -53,95 +51,11 @@ public static bool TryAttach(ITracer tracer, string root, out string errorMessag return true; } - public static bool IsHealthy(out string error, out string warning, ITracer tracer) - { - // TODO 1026787: Record errors\warnings in the event log - - warning = string.Empty; - - if (!IsServiceRunning(out error, tracer)) - { - return false; - } - - CheckTimeoutConfiguration(out warning, tracer); - return true; - } - - public static bool TryGetTimeout(out int timeoutMs, out string error, ITracer tracer = null) + public static bool IsHealthy(out string error, ITracer tracer) { - timeoutMs = 0; - error = string.Empty; - object value; - try - { - value = ProcessHelper.GetValueFromRegistry(GvFltParametersHive, GvFltParametersKey, GvFltTimeoutValue); - } - catch (UnauthorizedAccessException e) - { - if (tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", EtwArea); - metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "UnauthorizedAccessException while trying to read GvFlt timeout"); - tracer.RelatedError(metadata); - } - - error = "Failed to read GvFlt timeout from registry. " + e.Message; - return false; - } - catch (SecurityException e) - { - if (tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", EtwArea); - metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "SecurityException while trying to read GvFlt timeout"); - tracer.RelatedError(metadata); - } - - error = "Failed to read GvFlt timeout from registry. " + e.Message; - return false; - } - catch (Exception e) - { - if (tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", EtwArea); - metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "Exception while trying to read GvFlt timeout"); - tracer.RelatedError(metadata); - } - - error = "Failed to read GvFlt timeout from registry. " + e.Message; - return false; - } - - try - { - timeoutMs = Convert.ToInt32(value); - } - catch (Exception e) - { - if (tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", EtwArea); - metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "Exception while trying to convert GvFlt timeout to int"); - tracer.RelatedError(metadata); - } - - error = "GvFlt timeout not properly configured, failed to convert value to int: " + e.Message; - return false; - } - - return true; + return IsServiceRunning(out error, tracer); } - + private static bool IsServiceRunning(out string error, ITracer tracer) { error = string.Empty; @@ -159,8 +73,7 @@ private static bool IsServiceRunning(out string error, ITracer tracer) EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "InvalidOperationException: GvFlt Service was not found"); - tracer.RelatedError(metadata); + tracer.RelatedError(metadata, "InvalidOperationException: GvFlt Service was not found"); } error = "Error: GvFlt Service was not found. To resolve, re-install GVFS"; @@ -173,8 +86,7 @@ private static bool IsServiceRunning(out string error, ITracer tracer) { EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); - metadata.Add("ErrorMessage", "GvFlt Service is not running"); - tracer.RelatedError(metadata); + tracer.RelatedError(metadata, "GvFlt Service is not running"); } error = "Error: GvFlt Service is not running. To resolve, run \"sc start gvflt\" from an elevated command prompt"; @@ -182,22 +94,7 @@ private static bool IsServiceRunning(out string error, ITracer tracer) } return true; - } - - private static void CheckTimeoutConfiguration(out string warning, ITracer tracer) - { - warning = string.Empty; - string error; - int timemoutMs; - if (!TryGetTimeout(out timemoutMs, out error, tracer)) - { - warning = "Warning: Failed to validate GvFlt timeout configuration: " + error; - } - else if (timemoutMs < MinGvFltTimeoutMs) - { - warning = string.Format("Warning: GvFlt timeout not properly configured, timeout {0} less than recommended value {1}", timemoutMs, MinGvFltTimeoutMs); - } - } + } private static class NativeMethods { diff --git a/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs index 557443f1fe..1ab54d90dd 100644 --- a/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs @@ -1,7 +1,6 @@ using Microsoft.Win32.SafeHandles; using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices; namespace GVFS.Common.FileSystem { @@ -41,6 +40,11 @@ public virtual bool FileExists(string path) return File.Exists(path); } + public virtual bool DirectoryExists(string path) + { + return Directory.Exists(path); + } + public virtual void CopyFile(string sourcePath, string destinationPath, bool overwrite) { File.Copy(sourcePath, destinationPath, overwrite); @@ -50,7 +54,7 @@ public virtual void DeleteFile(string path) { File.Delete(path); } - + public virtual string ReadAllText(string path) { return File.ReadAllText(path); @@ -66,20 +70,22 @@ public virtual void WriteAllText(string path, string contents) File.WriteAllText(path, contents); } - public virtual Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode) + public Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode) { - return this.OpenFileStream(path, fileMode, fileAccess, NativeMethods.FileAttributes.FILE_ATTRIBUTE_NORMAL, shareMode); + return this.OpenFileStream(path, fileMode, fileAccess, shareMode, FileOptions.None); } - public virtual Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, NativeMethods.FileAttributes attributes, FileShare shareMode) + public virtual void MoveAndOverwriteFile(string sourceFileName, string destinationFilename) { - FileAccess access = fileAccess & FileAccess.ReadWrite; - return new FileStream((SafeFileHandle)this.OpenFile(path, fileMode, fileAccess, (FileAttributes)attributes, shareMode), access, DefaultStreamBufferSize, true); + NativeMethods.MoveFile( + sourceFileName, + destinationFilename, + NativeMethods.MoveFileFlags.MoveFileReplaceExisting | NativeMethods.MoveFileFlags.MoveFileCopyAllowed); } - public virtual SafeHandle OpenFile(string path, FileMode fileMode, FileAccess fileAccess, FileAttributes attributes, FileShare shareMode) + public virtual Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options) { - return NativeMethods.OpenFile(path, fileMode, (NativeMethods.FileAccess)fileAccess, shareMode, (NativeMethods.FileAttributes)attributes); + return new FileStream(path, fileMode, fileAccess, shareMode, DefaultStreamBufferSize, options); } public virtual void CreateDirectory(string path) @@ -139,5 +145,15 @@ public virtual FileProperties GetFileProperties(string path) return FileProperties.DefaultFile; } } + + public virtual void MoveFile(string sourcePath, string targetPath) + { + File.Move(sourcePath, targetPath); + } + + public virtual string[] GetFiles(string directoryPath, string mask) + { + return Directory.GetFiles(directoryPath, mask); + } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/GVFS.Common.csproj b/GVFS/GVFS.Common/GVFS.Common.csproj index f25911cea3..8f66b91084 100644 --- a/GVFS/GVFS.Common/GVFS.Common.csproj +++ b/GVFS/GVFS.Common/GVFS.Common.csproj @@ -39,22 +39,7 @@ MinimumRecommendedRules.ruleset true - - 0.2.173.2 - - - ..\..\..\packages\Microsoft.Database.Collections.Generic.1.9.4\lib\net40\Esent.Collections.dll - True - - - ..\..\..\packages\ManagedEsent.1.9.4\lib\net40\Esent.Interop.dll - True - - - ..\..\..\packages\Microsoft.Database.Isam.1.9.4\lib\net40\Esent.Isam.dll - True - False ..\..\..\packages\Microsoft.Diagnostics.Tracing.EventSource.Redist.1.1.28\lib\net40\Microsoft.Diagnostics.Tracing.EventSource.dll @@ -82,11 +67,17 @@ CommonAssemblyVersion.cs + + + + + + @@ -107,7 +98,6 @@ - @@ -117,12 +107,10 @@ - - @@ -149,7 +137,6 @@ - @@ -160,6 +147,7 @@ + @@ -182,7 +170,8 @@ - $(SolutionDir)\Scripts\CreateCommonAssemblyVersion.bat $(GVFSVersion) $(SolutionDir)\.. + + + \ No newline at end of file diff --git a/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs b/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs new file mode 100644 index 0000000000..97e54a3506 --- /dev/null +++ b/GVFS/GVFS.PerfProfiling/ProfilingEnvironment.cs @@ -0,0 +1,62 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Common.Http; +using GVFS.Common.Tracing; +using GVFS.GVFlt; + +namespace GVFS.PerfProfiling +{ + class ProfilingEnvironment + { + public ProfilingEnvironment(string enlistmentRootPath) + { + this.Enlistment = this.CreateEnlistment(enlistmentRootPath); + this.Context = this.CreateContext(); + this.GVFltCallbacks = this.CreateGVFltCallbacks(); + } + + public GVFSEnlistment Enlistment { get; private set; } + public GVFSContext Context { get; private set; } + public GVFltCallbacks GVFltCallbacks { get; private set; } + + private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) + { + string gitBinPath = GitProcess.GetInstalledGitBinPath(); + string hooksPath = ProcessHelper.WhereDirectory(GVFSConstants.GVFSHooksExecutableName); + + return GVFSEnlistment.CreateFromDirectory(enlistmentRootPath, gitBinPath, hooksPath); + } + + private GVFSContext CreateContext() + { + ITracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "GVFS.PerfProfiling", useCriticalTelemetryFlag: false); + + PhysicalFileSystem fileSystem = new PhysicalFileSystem(); + GitRepo gitRepo = new GitRepo( + tracer, + this.Enlistment, + fileSystem); + return new GVFSContext(tracer, fileSystem, gitRepo, this.Enlistment); + } + + private GVFltCallbacks CreateGVFltCallbacks() + { + string error; + if (!RepoMetadata.TryInitialize(this.Context.Tracer, this.Enlistment.DotGVFSRoot, out error)) + { + throw new InvalidRepoException(error); + } + + CacheServerInfo cacheServer = new CacheServerInfo(this.Context.Enlistment.RepoUrl, "None"); + GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor( + this.Context.Tracer, + this.Context.Enlistment, + cacheServer, + new RetryConfig()); + + GVFSGitObjects gitObjects = new GVFSGitObjects(this.Context, objectRequestor); + return new GVFltCallbacks(this.Context, gitObjects, RepoMetadata.Instance); + } + } +} diff --git a/GVFS/GVFS.PerfProfiling/Program.cs b/GVFS/GVFS.PerfProfiling/Program.cs new file mode 100644 index 0000000000..ea1a1b745e --- /dev/null +++ b/GVFS/GVFS.PerfProfiling/Program.cs @@ -0,0 +1,49 @@ +using GVFS.Common; +using GVFS.GVFlt.DotGit; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; + +namespace GVFS.PerfProfiling +{ + class Program + { + static void Main(string[] args) + { + ProfilingEnvironment environment = new ProfilingEnvironment(@"M:\OS"); + TimeIt( + "Validate Index", + () => GitIndexProjection.ReadIndex(Path.Combine(environment.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Index))); + TimeIt( + "Index Parse (new projection)", + () => environment.GVFltCallbacks.GitIndexProjectionProfiler.ForceRebuildProjection()); + TimeIt( + "Index Parse (update offsets and validate)", + () => environment.GVFltCallbacks.GitIndexProjectionProfiler.ForceUpdateOffsetsAndValidateSparseCheckout()); + TimeIt( + "Index Parse (validate sparse checkout)", + () => environment.GVFltCallbacks.GitIndexProjectionProfiler.ForceValidateSparseCheckout()); + Console.WriteLine("Press Enter to exit"); + } + + private static void TimeIt(string name, Action action) + { + List times = new List(); + + for (int i = 0; i < 10; i++) + { + Stopwatch stopwatch = Stopwatch.StartNew(); + action(); + stopwatch.Stop(); + + times.Add(stopwatch.Elapsed); + Console.WriteLine(stopwatch.Elapsed.TotalMilliseconds); + } + + Console.WriteLine("Average Time - " + name + times.Select(timespan => timespan.TotalMilliseconds).Average()); + Console.WriteLine(); + } + } +} diff --git a/GVFS/GVFS.PerfProfiling/Properties/AssemblyInfo.cs b/GVFS/GVFS.PerfProfiling/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..49d51708d2 --- /dev/null +++ b/GVFS/GVFS.PerfProfiling/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("GVFS.PerfProfiling")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("GVFS.PerfProfiling")] +[assembly: AssemblyCopyright("Copyright © 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("c5d3ca26-562f-4ca4-a378-b93e97a730e3")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/GVFS/GVFS.ReadObjectHook/GVFS.ReadObjectHook.vcxproj b/GVFS/GVFS.ReadObjectHook/GVFS.ReadObjectHook.vcxproj index 681836de4f..466ca234c7 100644 --- a/GVFS/GVFS.ReadObjectHook/GVFS.ReadObjectHook.vcxproj +++ b/GVFS/GVFS.ReadObjectHook/GVFS.ReadObjectHook.vcxproj @@ -30,9 +30,6 @@ true MultiByte - - 0.2.173.2 - @@ -74,7 +71,6 @@ $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log - $(SolutionDir)\Scripts\CreateCommonVersionHeader.bat $(GVFSVersion) $(SolutionDir)\.. $(SolutionDir)\..\BuildOutput @@ -103,7 +99,6 @@ $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log - $(SolutionDir)\Scripts\CreateCommonVersionHeader.bat $(GVFSVersion) $(SolutionDir)\.. $(SolutionDir)\..\BuildOutput diff --git a/GVFS/GVFS.ReadObjectHook/Version.rc b/GVFS/GVFS.ReadObjectHook/Version.rc index e10e4c5f16..d4ef5a44fb 100644 Binary files a/GVFS/GVFS.ReadObjectHook/Version.rc and b/GVFS/GVFS.ReadObjectHook/Version.rc differ diff --git a/GVFS/GVFS.Service/Configuration.cs b/GVFS/GVFS.Service/Configuration.cs index 4cb72d2601..6cc8f53a67 100644 --- a/GVFS/GVFS.Service/Configuration.cs +++ b/GVFS/GVFS.Service/Configuration.cs @@ -10,7 +10,7 @@ public class Configuration private Configuration() { - this.GVFSMountLocation = Path.Combine(AssemblyPath, GVFSConstants.MountExecutableName); + this.GVFSLocation = Path.Combine(AssemblyPath, GVFSConstants.GVFSExecutableName); this.GVFSServiceUILocation = Path.Combine(AssemblyPath, GVFSConstants.Service.UIName + GVFSConstants.ExecutableExtension); } @@ -34,8 +34,8 @@ public static string AssemblyPath return assemblyPath; } } - - public string GVFSMountLocation { get; private set; } + + public string GVFSLocation { get; private set; } public string GVFSServiceUILocation { get; private set; } } } diff --git a/GVFS/GVFS.Service/GVFS.Service.csproj b/GVFS/GVFS.Service/GVFS.Service.csproj index 622d1e99ac..a591c5e46a 100644 --- a/GVFS/GVFS.Service/GVFS.Service.csproj +++ b/GVFS/GVFS.Service/GVFS.Service.csproj @@ -88,12 +88,14 @@ Component + + diff --git a/GVFS/GVFS.Service/GVFSMountProcess.cs b/GVFS/GVFS.Service/GVFSMountProcess.cs index f741a3dffe..052a822458 100644 --- a/GVFS/GVFS.Service/GVFSMountProcess.cs +++ b/GVFS/GVFS.Service/GVFSMountProcess.cs @@ -1,7 +1,8 @@ using GVFS.Common; using GVFS.Common.FileSystem; +using GVFS.Common.Git; using GVFS.Common.Tracing; -using Microsoft.Diagnostics.Tracing; +using GVFS.Service.Handlers; using System; namespace GVFS.Service @@ -23,13 +24,16 @@ public GVFSMountProcess(ITracer tracer, int sessionId) public bool Mount(string repoRoot) { string error; - string warning; - if (!GvFltFilter.IsHealthy(out error, out warning, this.tracer)) + if (!GvFltFilter.IsHealthy(out error, this.tracer)) { return false; } - this.CheckAntiVirusExclusion(this.tracer, repoRoot); + // Ensure the repo is excluded from antivirus before calling 'gvfs mount' + // to reduce chatter between GVFS.exe and GVFS.Service.exe + string errorMessage; + bool isExcluded; + ExcludeFromAntiVirusHandler.CheckAntiVirusExclusion(this.tracer, repoRoot, out isExcluded, out errorMessage); string unusedMessage; if (!GvFltFilter.TryAttach(this.tracer, repoRoot, out unusedMessage)) @@ -39,12 +43,11 @@ public bool Mount(string repoRoot) if (!this.CallGVFSMount(repoRoot)) { - this.tracer.RelatedError("Unable to start the GVFS.Mount process."); + this.tracer.RelatedError("Unable to start the GVFS.exe process."); return false; } - string errorMessage; - if (!GVFSEnlistment.WaitUntilMounted(repoRoot, out errorMessage)) + if (!GVFSEnlistment.WaitUntilMounted(repoRoot, false, out errorMessage)) { this.tracer.RelatedError(errorMessage); return false; @@ -64,27 +67,7 @@ public void Dispose() private bool CallGVFSMount(string repoRoot) { - return this.CurrentUser.RunAs(Configuration.Instance.GVFSMountLocation, repoRoot); - } - - private void CheckAntiVirusExclusion(ITracer tracer, string path) - { - string errorMessage; - bool isExcluded; - if (AntiVirusExclusions.TryGetIsPathExcluded(path, out isExcluded, out errorMessage)) - { - if (!isExcluded) - { - if (!AntiVirusExclusions.AddAntiVirusExclusion(path, out errorMessage)) - { - tracer.RelatedError("Could not add this repo to the antivirus exclusion list. Error: {0}", errorMessage); - } - } - } - else - { - tracer.RelatedError("Unable to determine if this repo is excluded from antivirus. Error: {0}", errorMessage); - } + return this.CurrentUser.RunAs(Configuration.Instance.GVFSLocation, "mount " + repoRoot); } } } diff --git a/GVFS/GVFS.Service/GvfsService.cs b/GVFS/GVFS.Service/GvfsService.cs index e4117d8165..e5719d3f92 100644 --- a/GVFS/GVFS.Service/GvfsService.cs +++ b/GVFS/GVFS.Service/GvfsService.cs @@ -93,18 +93,21 @@ protected override void OnSessionChange(SessionChangeDescription changeDescripti { base.OnSessionChange(changeDescription); - if (changeDescription.Reason == SessionChangeReason.SessionLogon) + if (!GVFSEnlistment.IsUnattended(tracer: null)) { - this.tracer.RelatedInfo("SessionLogon detected, sessionId: {0}", changeDescription.SessionId); - using (ITracer activity = this.tracer.StartActivity("LogonAutomount", EventLevel.Informational)) + if (changeDescription.Reason == SessionChangeReason.SessionLogon) { - this.repoRegistry.AutoMountRepos(changeDescription.SessionId); - this.repoRegistry.TraceStatus(); + this.tracer.RelatedInfo("SessionLogon detected, sessionId: {0}", changeDescription.SessionId); + using (ITracer activity = this.tracer.StartActivity("LogonAutomount", EventLevel.Informational)) + { + this.repoRegistry.AutoMountRepos(changeDescription.SessionId); + this.repoRegistry.TraceStatus(); + } + } + else if (changeDescription.Reason == SessionChangeReason.SessionLogoff) + { + this.tracer.RelatedInfo("SessionLogoff detected"); } - } - else if (changeDescription.Reason == SessionChangeReason.SessionLogoff) - { - this.tracer.RelatedInfo("SessionLogoff detected"); } } catch (Exception e) @@ -237,12 +240,25 @@ private void HandleRequest(ITracer tracer, string request, NamedPipeServer.Conne break; + case NamedPipeMessages.ExcludeFromAntiVirusRequest.Header: + try + { + NamedPipeMessages.ExcludeFromAntiVirusRequest excludeFromAntiVirusRequest = NamedPipeMessages.ExcludeFromAntiVirusRequest.FromMessage(message); + ExcludeFromAntiVirusHandler excludeHandler = new ExcludeFromAntiVirusHandler(activity, connection, excludeFromAntiVirusRequest); + excludeHandler.Run(); + } + catch (SerializationException ex) + { + activity.RelatedError("Could not deserialize exclude from antivirus request: {0}", ex.Message); + } + + break; + default: EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); metadata.Add("Header", message.Header); - metadata.Add("ErrorMessage", "HandleNewConnection: Unknown request"); - this.tracer.RelatedError(metadata); + this.tracer.RelatedWarning(metadata, "HandleNewConnection: Unknown request", Keywords.Telemetry); connection.TrySendResponse(NamedPipeMessages.UnknownRequest); break; @@ -255,8 +271,7 @@ private void LogExceptionAndExit(Exception e, string method) EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "Unhandled exception in " + method); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, "Unhandled exception in " + method); Environment.Exit((int)ReturnCode.GenericError); } } diff --git a/GVFS/GVFS.Service/Handlers/ExcludeFromAntiVirusHandler.cs b/GVFS/GVFS.Service/Handlers/ExcludeFromAntiVirusHandler.cs new file mode 100644 index 0000000000..03ea656279 --- /dev/null +++ b/GVFS/GVFS.Service/Handlers/ExcludeFromAntiVirusHandler.cs @@ -0,0 +1,77 @@ +using GVFS.Common; +using GVFS.Common.NamedPipes; +using GVFS.Common.Tracing; + +namespace GVFS.Service.Handlers +{ + public class ExcludeFromAntiVirusHandler + { + private NamedPipeServer.Connection connection; + private NamedPipeMessages.ExcludeFromAntiVirusRequest request; + private ITracer tracer; + + public ExcludeFromAntiVirusHandler( + ITracer tracer, + NamedPipeServer.Connection connection, + NamedPipeMessages.ExcludeFromAntiVirusRequest request) + { + this.tracer = tracer; + this.connection = connection; + this.request = request; + } + + public static void CheckAntiVirusExclusion(ITracer tracer, string path, out bool isExcluded, out string errorMessage) + { + errorMessage = string.Empty; + if (AntiVirusExclusions.TryGetIsPathExcluded(path, out isExcluded, out errorMessage)) + { + if (!isExcluded) + { + if (AntiVirusExclusions.AddAntiVirusExclusion(path, out errorMessage)) + { + if (!AntiVirusExclusions.TryGetIsPathExcluded(path, out isExcluded, out errorMessage)) + { + errorMessage = string.Format("Unable to determine if this repo is excluded from antivirus after adding exclusion: {0}", errorMessage); + tracer.RelatedWarning(errorMessage); + } + } + else + { + errorMessage = string.Format("Could not add this repo to the antivirus exclusion list: {0}", errorMessage); + tracer.RelatedWarning(errorMessage); + } + } + } + else + { + errorMessage = string.Format("Unable to determine if this repo is excluded from antivirus: {0}", errorMessage); + tracer.RelatedWarning(errorMessage); + } + } + + public void Run() + { + string errorMessage; + NamedPipeMessages.CompletionState state = NamedPipeMessages.CompletionState.Success; + + bool isExcluded; + CheckAntiVirusExclusion(this.tracer, this.request.ExclusionPath, out isExcluded, out errorMessage); + + if (!isExcluded) + { + state = NamedPipeMessages.CompletionState.Failure; + } + + this.WriteToClient(new NamedPipeMessages.ExcludeFromAntiVirusRequest.Response() { State = state, ErrorMessage = errorMessage }); + } + + private void WriteToClient(NamedPipeMessages.ExcludeFromAntiVirusRequest.Response response) + { + NamedPipeMessages.Message message = response.ToMessage(); + if (!this.connection.TrySendResponse(message)) + { + this.tracer.RelatedError("Failed to send line to client: {0}", message); + } + } + } +} diff --git a/GVFS/GVFS.Common/RepoRegistration.cs b/GVFS/GVFS.Service/RepoRegistration.cs similarity index 91% rename from GVFS/GVFS.Common/RepoRegistration.cs rename to GVFS/GVFS.Service/RepoRegistration.cs index f8735512e0..ba95b8956d 100644 --- a/GVFS/GVFS.Common/RepoRegistration.cs +++ b/GVFS/GVFS.Service/RepoRegistration.cs @@ -1,6 +1,6 @@ using Newtonsoft.Json; -namespace GVFS.Common +namespace GVFS.Service { public class RepoRegistration { @@ -33,7 +33,7 @@ public override string ToString() { return string.Format( - "({0} - {1},{2}) {3}", + "({0} - {1}) {2}", this.IsActive ? "Active" : "Inactive", this.OwnerSID, this.EnlistmentRoot); diff --git a/GVFS/GVFS.Service/RepoRegistry.cs b/GVFS/GVFS.Service/RepoRegistry.cs index 4dff09b5d4..f461cbe145 100644 --- a/GVFS/GVFS.Service/RepoRegistry.cs +++ b/GVFS/GVFS.Service/RepoRegistry.cs @@ -29,7 +29,7 @@ public RepoRegistry(ITracer tracer, string serviceDataLocation) EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); metadata.Add("registryParentFolderPath", this.registryParentFolderPath); - metadata.Add("Message", "RepoRegistry created"); + metadata.Add(TracingConstants.MessageKey.InfoMessage, "RepoRegistry created"); this.tracer.RelatedEvent(EventLevel.Informational, "RepoRegistry_Created", metadata); } @@ -96,7 +96,7 @@ public void TraceStatus() } catch (Exception e) { - this.tracer.RelatedError("Error while tracing repos", e.ToString()); + this.tracer.RelatedError("Error while tracing repos: {0}", e.ToString()); } } @@ -124,7 +124,7 @@ public bool TryDeactivateRepo(string repoRoot, out string errorMessage) else { errorMessage = string.Format("Attempted to deactivate non-existent repo at '{0}'", repoRoot); - this.tracer.RelatedError(errorMessage); + this.tracer.RelatedWarning(errorMessage, Keywords.Telemetry); } } } @@ -190,8 +190,7 @@ public void AutoMountRepos(int sessionId) metadata.Add("Area", EtwArea); metadata.Add("OnDiskVersion", versionString); metadata.Add("ExpectedVersion", versionString); - metadata.Add("ErrorMessage", "ReadRegistry: Unsupported version"); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, "ReadRegistry: Unsupported version"); } return allRepos; @@ -213,8 +212,7 @@ public void AutoMountRepos(int sessionId) metadata.Add("Area", EtwArea); metadata.Add("entry", entry); metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "ReadRegistry: Failed to read entry"); - this.tracer.RelatedError(metadata); + this.tracer.RelatedError(metadata, "ReadRegistry: Failed to read entry"); } } } diff --git a/GVFS/GVFS.Tests/NUnitRunner.cs b/GVFS/GVFS.Tests/NUnitRunner.cs index 34bed1facf..16e8bf3fad 100644 --- a/GVFS/GVFS.Tests/NUnitRunner.cs +++ b/GVFS/GVFS.Tests/NUnitRunner.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Reflection; using System.Threading; @@ -18,6 +19,18 @@ public NUnitRunner(string[] args) this.excludedCategories = new List(); } + public string GetCustomArgWithParam(string arg) + { + string match = this.args.Where(a => a.StartsWith(arg + "=")).SingleOrDefault(); + if (match == null) + { + return null; + } + + this.args.Remove(match); + return match.Substring(arg.Length + 1); + } + public bool HasCustomArg(string arg) { // We also remove it as we're checking, because nunit wouldn't understand what it means diff --git a/GVFS/GVFS.Tests/Should/StringShouldExtensions.cs b/GVFS/GVFS.Tests/Should/StringShouldExtensions.cs index b46be5d826..1905dd56a1 100644 --- a/GVFS/GVFS.Tests/Should/StringShouldExtensions.cs +++ b/GVFS/GVFS.Tests/Should/StringShouldExtensions.cs @@ -5,6 +5,13 @@ namespace GVFS.Tests.Should { public static class StringShouldExtensions { + public static int ShouldBeAnInt(this string value, string message) + { + int output; + Assert.IsTrue(int.TryParse(value, out output), message); + return output; + } + public static string ShouldContain(this string actualValue, params string[] expectedSubstrings) { foreach (string expectedSubstring in expectedSubstrings) diff --git a/GVFS/GVFS.UnitTests/Common/BackgroundGitUpdateQueueTests.cs b/GVFS/GVFS.UnitTests/Common/BackgroundGitUpdateQueueTests.cs new file mode 100644 index 0000000000..90498c3ab2 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/BackgroundGitUpdateQueueTests.cs @@ -0,0 +1,182 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.GVFlt; +using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Mock; +using NUnit.Framework; +using System.IO; +using System.Text; +using static GVFS.GVFlt.GVFltCallbacks; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class BackgroundGitUpdateQueueTests + { + private const string MockEntryFileName = "mock:\\entries.dat"; + + private const string NonAsciiString = @"ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك"; + + private const string Item1EntryText = "A 1\00\0mock:\\VirtualPath\0" + NonAsciiString + "\r\n"; + private const string Item2EntryText = "A 2\01\0mock:\\VirtualPath2\0mock:\\OldVirtualPath2\r\n"; + + private const string CorruptEntryText = Item1EntryText + "A 1\0\"item1"; + + private static readonly BackgroundGitUpdate Item1Payload = new BackgroundGitUpdate(BackgroundGitUpdate.OperationType.Invalid, "mock:\\VirtualPath", NonAsciiString); + private static readonly BackgroundGitUpdate Item2Payload = new BackgroundGitUpdate(BackgroundGitUpdate.OperationType.OnFileCreated, "mock:\\VirtualPath2", "mock:\\OldVirtualPath2"); + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void ReturnsFalseWhenOpenFails() + { + MockFileSystem fs = new MockFileSystem(); + fs.File = new ReusableMemoryStream(string.Empty); + fs.ThrowDuringOpen = true; + + string error; + BackgroundGitUpdateQueue dut; + BackgroundGitUpdateQueue.TryCreate(null, MockEntryFileName, fs, out dut, out error).ShouldEqual(false); + dut.ShouldBeNull(); + error.ShouldNotBeNull(); + } + + [TestCase] + public void TryPeekDoesNotDequeue() + { + MockFileSystem fs = new MockFileSystem(); + BackgroundGitUpdateQueue dut = CreateFileBasedQueue(fs, Item1EntryText); + + for (int i = 0; i < 5; ++i) + { + BackgroundGitUpdate item; + dut.TryPeek(out item).ShouldEqual(true); + item.ShouldEqual(Item1Payload); + } + + fs.File.ReadAsString().ShouldEqual(Item1EntryText); + } + + [TestCase] + public void StoresAddRecord() + { + MockFileSystem fs = new MockFileSystem(); + BackgroundGitUpdateQueue dut = CreateFileBasedQueue(fs, string.Empty); + + dut.EnqueueAndFlush(Item1Payload); + + fs.File.ReadAsString().ShouldEqual(Item1EntryText); + } + + [TestCase] + public void TruncatesWhenEmpty() + { + MockFileSystem fs = new MockFileSystem(); + BackgroundGitUpdateQueue dut = CreateFileBasedQueue(fs, Item1EntryText); + + dut.DequeueAndFlush(Item1Payload); + + fs.File.Length.ShouldEqual(0); + } + + [TestCase] + public void RecoversWhenCorrupt() + { + MockFileSystem fs = new MockFileSystem(); + BackgroundGitUpdateQueue dut = CreateFileBasedQueue(fs, CorruptEntryText); + + fs.File.ReadAsString().ShouldEqual(Item1EntryText); + dut.Count.ShouldEqual(1); + } + + [TestCase] + public void StoresDeleteRecord() + { + const string DeleteRecord = "D 1\r\n"; + + MockFileSystem fs = new MockFileSystem(); + BackgroundGitUpdateQueue dut = CreateFileBasedQueue(fs, Item1EntryText); + + // Add a second entry to keep FileBasedQueue from setting the stream length to 0 + dut.EnqueueAndFlush(Item2Payload); + + fs.File.ReadAsString().ShouldEqual(Item1EntryText + Item2EntryText); + fs.File.ReadAt(fs.File.Length - 2, 2).ShouldEqual("\r\n"); + + dut.DequeueAndFlush(Item1Payload); + dut.Count.ShouldEqual(1); + + BackgroundGitUpdate item; + dut.TryPeek(out item).ShouldEqual(true); + item.ShouldEqual(Item2Payload); + + fs.File.Length.ShouldEqual(Encoding.UTF8.GetByteCount(Item1EntryText) + Item2EntryText.Length + DeleteRecord.Length); + fs.File.ReadAt(Encoding.UTF8.GetByteCount(Item1EntryText) + Item2EntryText.Length, DeleteRecord.Length).ShouldEqual(DeleteRecord); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void WrapsIOExceptionsDuringWrite() + { + MockFileSystem fs = new MockFileSystem(); + BackgroundGitUpdateQueue dut = CreateFileBasedQueue(fs, Item1EntryText); + + fs.File.TruncateWrites = true; + + Assert.Throws(() => dut.EnqueueAndFlush(Item2Payload)); + + fs.File.TruncateWrites = false; + fs.File.ReadAt(fs.File.Length - 2, 2).ShouldNotEqual("\r\n", "Bad Test: The file is supposed to be corrupt."); + + string error; + BackgroundGitUpdateQueue.TryCreate(null, MockEntryFileName, fs, out dut, out error).ShouldEqual(true); + using (dut) + { + BackgroundGitUpdate output; + dut.TryPeek(out output).ShouldEqual(true); + output.ShouldEqual(Item1Payload); + dut.DequeueAndFlush(output); + } + } + + private static BackgroundGitUpdateQueue CreateFileBasedQueue(MockFileSystem fs, string initialContents) + { + fs.File = new ReusableMemoryStream(initialContents); + fs.ExpectedPath = MockEntryFileName; + + string error; + BackgroundGitUpdateQueue dut; + BackgroundGitUpdateQueue.TryCreate(null, MockEntryFileName, fs, out dut, out error).ShouldEqual(true, error); + dut.ShouldNotBeNull(); + return dut; + } + + private class MockFileSystem : PhysicalFileSystem + { + public bool ThrowDuringOpen { get; set; } + + public string ExpectedPath { get; set; } + public ReusableMemoryStream File { get; set; } + + public override void CreateDirectory(string path) + { + } + + public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options) + { + if (this.ThrowDuringOpen) + { + throw new IOException("Test Error"); + } + + path.ShouldEqual(this.ExpectedPath); + return this.File; + } + + public override bool FileExists(string path) + { + return true; + } + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerInfoTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerInfoTests.cs deleted file mode 100644 index 4200e7887c..0000000000 --- a/GVFS/GVFS.UnitTests/Common/CacheServerInfoTests.cs +++ /dev/null @@ -1,200 +0,0 @@ -using GVFS.Common.Git; -using GVFS.Common.Http; -using GVFS.Tests.Should; -using NUnit.Framework; -using System.Collections.Generic; -using GVFS.UnitTests.Mock.Common; -using GVFS.UnitTests.Mock.Git; - -namespace GVFS.UnitTests.Common -{ - [TestFixture] - public class CacheServerInfoTests - { - private const string DefaultCacheName = "DefaultCache"; - private const string UserSuppliedCacheName = "ValidCache"; - private const string UserSuppliedUrl = "https://validUrl"; - - private static readonly IEnumerable KnownCaches = new List() - { - new CacheServerInfo("https://anotherValidUrl", DefaultCacheName, true), - new CacheServerInfo(UserSuppliedUrl, UserSuppliedCacheName, false) - }; - - private MockEnlistment enlistment = new MockEnlistment(); - - [TestCase] - public void ParsesValidUserSuppliedUrl() - { - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - UserSuppliedUrl, - gitProcess: null, - enlistment: this.enlistment, - knownCaches: null, - output: out output, - error: out error).ShouldBeTrue(error); - output.Url.ShouldEqual(UserSuppliedUrl); - } - - [TestCase] - public void FailsToParseInvalidUserSuppliedUrl() - { - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - "invalidCacheUrl", - gitProcess: null, - enlistment: this.enlistment, - knownCaches: null, - output: out output, - error: out error).ShouldBeFalse(); - output.ShouldBeNull(); - } - - [TestCase] - public void ParsesUserSuppliedFriendlyName() - { - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - UserSuppliedCacheName, - gitProcess: null, - enlistment: this.enlistment, - knownCaches: KnownCaches, - output: out output, - error: out error).ShouldBeTrue(error); - output.Url.ShouldEqual(UserSuppliedUrl); - } - - [TestCase] - public void FailsToParseInvalidUserSuppliedFriendlyName() - { - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - "invalidCacheName", - gitProcess: null, - enlistment: this.enlistment, - knownCaches: KnownCaches, - output: out output, - error: out error).ShouldBeFalse(); - output.ShouldBeNull(); - } - - [TestCase] - public void ParsesConfiguredCacheName() - { - MockGitProcess git = new MockGitProcess(); - git.SetExpectedCommandResult("config gvfs.cache-server", () => new GitProcess.Result(UserSuppliedCacheName, string.Empty, GitProcess.Result.SuccessCode)); - - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - userUrlish: null, - gitProcess: git, - enlistment: this.enlistment, - knownCaches: KnownCaches, - output: out output, - error: out error).ShouldBeTrue(error); - output.Url.ShouldEqual(UserSuppliedUrl); - } - - [TestCase] - public void ResolvesUrlIntoNone() - { - MockGitProcess git = new MockGitProcess(); - - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - userUrlish: this.enlistment.RepoUrl, - gitProcess: git, - enlistment: this.enlistment, - knownCaches: KnownCaches, - output: out output, - error: out error).ShouldBeTrue(error); - - output.Name.ShouldEqual(CacheServerInfo.NoneFriendlyName); - output.Url.ShouldEqual(this.enlistment.RepoUrl); - } - - [TestCase] - public void ResolvesUrlIntoFriendlyName() - { - MockGitProcess git = new MockGitProcess(); - - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - userUrlish: UserSuppliedUrl, - gitProcess: git, - enlistment: this.enlistment, - knownCaches: KnownCaches, - output: out output, - error: out error).ShouldBeTrue(error); - - output.Name.ShouldEqual(UserSuppliedCacheName); - output.Url.ShouldEqual(UserSuppliedUrl); - } - - [TestCase] - public void FallsBackToDeprecatedConfigSetting() - { - MockGitProcess git = new MockGitProcess(); - git.SetExpectedCommandResult(@"config gvfs.mock:\repourl.cache-server-url", () => new GitProcess.Result(UserSuppliedUrl, string.Empty, GitProcess.Result.SuccessCode)); - git.SetExpectedCommandResult(@"config --local gvfs.cache-server " + UserSuppliedUrl, () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - userUrlish: null, - gitProcess: git, - enlistment: this.enlistment, - knownCaches: null, - output: out output, - error: out error).ShouldBeTrue(error); - - output.Url.ShouldEqual(UserSuppliedUrl); - } - - [TestCase] - public void FallsBackToDefaultCache() - { - MockGitProcess git = new MockGitProcess(); - git.SetExpectedCommandResult(@"config gvfs.mock:\repourl.cache-server-url", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.GenericFailureCode)); - - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - userUrlish: null, - gitProcess: git, - enlistment: this.enlistment, - knownCaches: KnownCaches, - output: out output, - error: out error).ShouldBeTrue(error); - - output.Name.ShouldEqual(DefaultCacheName); - } - - [TestCase] - public void FallsBackToNone() - { - MockGitProcess git = new MockGitProcess(); - git.SetExpectedCommandResult(@"config gvfs.mock:\repourl.cache-server-url", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.GenericFailureCode)); - - string error; - CacheServerInfo output; - CacheServerInfo.TryDetermineCacheServer( - userUrlish: null, - gitProcess: git, - enlistment: this.enlistment, - knownCaches: null, - output: out output, - error: out error).ShouldBeTrue(error); - - output.Name.ShouldEqual(CacheServerInfo.NoneFriendlyName); - output.Url.ShouldEqual(this.enlistment.RepoUrl); - } - } -} diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs new file mode 100644 index 0000000000..e45ee742b4 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs @@ -0,0 +1,176 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Http; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.Git; +using NUnit.Framework; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class CacheServerResolverTests + { + private const string CacheServerUrl = "https://cache/server"; + private const string CacheServerName = "TestCacheServer"; + + private const string NoneFriendlyName = "None"; + private const string DefaultFriendlyName = "Default"; + private const string UserDefinedFriendlyName = "User Defined"; + + [TestCase] + public void CanGetCacheServerFromNewConfig() + { + MockEnlistment enlistment = this.CreateEnlistment(CacheServerUrl); + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment); + + cacheServer.Url.ShouldEqual(CacheServerUrl); + CacheServerResolver.GetUrlFromConfig(enlistment).ShouldEqual(CacheServerUrl); + } + + [TestCase] + public void CanGetCacheServerFromOldConfig() + { + MockEnlistment enlistment = this.CreateEnlistment(null, CacheServerUrl); + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment); + + cacheServer.Url.ShouldEqual(CacheServerUrl); + CacheServerResolver.GetUrlFromConfig(enlistment).ShouldEqual(CacheServerUrl); + } + + [TestCase] + public void CanGetCacheServerWithNoConfig() + { + MockEnlistment enlistment = this.CreateEnlistment(); + + this.ValidateIsNone(enlistment, CacheServerResolver.GetCacheServerFromConfig(enlistment)); + CacheServerResolver.GetUrlFromConfig(enlistment).ShouldEqual(enlistment.RepoUrl); + } + + [TestCase] + public void CanResolveUrlForKnownName() + { + CacheServerResolver resolver = this.CreateResolver(); + + CacheServerInfo resolvedCacheServer; + string error; + resolver.TryResolveUrlFromRemote(CacheServerName, this.CreateGVFSConfig(), out resolvedCacheServer, out error); + + resolvedCacheServer.Url.ShouldEqual(CacheServerUrl); + resolvedCacheServer.Name.ShouldEqual(CacheServerName); + } + + [TestCase] + public void CanResolveNameFromKnownUrl() + { + CacheServerResolver resolver = this.CreateResolver(); + CacheServerInfo resolvedCacheServer = resolver.ResolveNameFromRemote(CacheServerUrl, this.CreateGVFSConfig()); + + resolvedCacheServer.Url.ShouldEqual(CacheServerUrl); + resolvedCacheServer.Name.ShouldEqual(CacheServerName); + } + + [TestCase] + public void CanResolveNameFromCustomUrl() + { + const string CustomUrl = "https://not/a/known/cache/server"; + + CacheServerResolver resolver = this.CreateResolver(); + CacheServerInfo resolvedCacheServer = resolver.ResolveNameFromRemote(CustomUrl, this.CreateGVFSConfig()); + + resolvedCacheServer.Url.ShouldEqual(CustomUrl); + resolvedCacheServer.Name.ShouldEqual(UserDefinedFriendlyName); + } + + [TestCase] + public void CanParseUrl() + { + CacheServerResolver resolver = new CacheServerResolver(new MockTracer(), this.CreateEnlistment()); + CacheServerInfo parsedCacheServer = resolver.ParseUrlOrFriendlyName(CacheServerUrl); + + parsedCacheServer.Url.ShouldEqual(CacheServerUrl); + parsedCacheServer.Name.ShouldEqual(null); + } + + [TestCase] + public void CanParseName() + { + CacheServerResolver resolver = new CacheServerResolver(new MockTracer(), this.CreateEnlistment()); + CacheServerInfo parsedCacheServer = resolver.ParseUrlOrFriendlyName(CacheServerName); + + parsedCacheServer.Url.ShouldEqual(null); + parsedCacheServer.Name.ShouldEqual(CacheServerName); + } + + [TestCase] + public void CanParseAndResolveDefault() + { + CacheServerResolver resolver = this.CreateResolver(); + + CacheServerInfo parsedCacheServer = resolver.ParseUrlOrFriendlyName(null); + parsedCacheServer.Url.ShouldEqual(null); + parsedCacheServer.Name.ShouldEqual(DefaultFriendlyName); + + CacheServerInfo resolvedCacheServer; + string error; + resolver.TryResolveUrlFromRemote(parsedCacheServer.Name, this.CreateGVFSConfig(), out resolvedCacheServer, out error); + + resolvedCacheServer.Url.ShouldEqual(CacheServerUrl); + resolvedCacheServer.Name.ShouldEqual(CacheServerName); + } + + [TestCase] + public void CanParseAndResolveNoCacheServer() + { + MockEnlistment enlistment = this.CreateEnlistment(); + CacheServerResolver resolver = this.CreateResolver(enlistment); + + this.ValidateIsNone(enlistment, resolver.ParseUrlOrFriendlyName(NoneFriendlyName)); + this.ValidateIsNone(enlistment, resolver.ParseUrlOrFriendlyName(enlistment.RepoUrl)); + + CacheServerInfo resolvedCacheServer; + string error; + resolver.TryResolveUrlFromRemote(NoneFriendlyName, this.CreateGVFSConfig(), out resolvedCacheServer, out error) + .ShouldEqual(false, "Should not succeed in resolving the name 'None'"); + + resolvedCacheServer.ShouldEqual(null); + error.ShouldNotBeNull(); + } + + private void ValidateIsNone(Enlistment enlistment, CacheServerInfo cacheServer) + { + cacheServer.Url.ShouldEqual(enlistment.RepoUrl); + cacheServer.Name.ShouldEqual(NoneFriendlyName); + } + + private MockEnlistment CreateEnlistment(string newConfigValue = null, string oldConfigValue = null) + { + MockGitProcess gitProcess = new MockGitProcess(); + gitProcess.SetExpectedCommandResult( + "config --local gvfs.cache-server", + () => new GitProcess.Result(newConfigValue ?? string.Empty, string.Empty, newConfigValue != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + gitProcess.SetExpectedCommandResult( + "config gvfs.mock:\\repourl.cache-server-url", + () => new GitProcess.Result(oldConfigValue ?? string.Empty, string.Empty, oldConfigValue != null ? GitProcess.Result.SuccessCode : GitProcess.Result.GenericFailureCode)); + + return new MockEnlistment(gitProcess); + } + + private GVFSConfig CreateGVFSConfig() + { + return new GVFSConfig + { + CacheServers = new[] + { + new CacheServerInfo(CacheServerUrl, CacheServerName, globalDefault: true), + } + }; + } + + private CacheServerResolver CreateResolver(MockEnlistment enlistment = null) + { + enlistment = enlistment ?? this.CreateEnlistment(); + return new CacheServerResolver(new MockTracer(), enlistment); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/FileBasedDictionaryTests.cs b/GVFS/GVFS.UnitTests/Common/FileBasedDictionaryTests.cs new file mode 100644 index 0000000000..50d1c0f025 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/FileBasedDictionaryTests.cs @@ -0,0 +1,329 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Mock; +using GVFS.UnitTests.Mock.FileSystem; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class FileBasedDictionaryTests + { + private const string MockEntryFileName = "mock:\\entries.dat"; + + private const string TestKey = "akey"; + private const string TestValue = "avalue"; + private const string UpdatedTestValue = "avalue2"; + + private const string TestEntry = "A {\"Key\":\"akey\",\"Value\":\"avalue\"}\r\n"; + private const string UpdatedTestEntry = "A {\"Key\":\"akey\",\"Value\":\"avalue2\"}\r\n"; + + [TestCase] + public void ParsesExistingDataCorrectly() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(); + FileBasedDictionary dut = CreateFileBasedDictionary(fs, TestEntry); + + string value; + dut.TryGetValue(TestKey, out value).ShouldEqual(true); + value.ShouldEqual(TestValue); + } + + [TestCase] + public void SetValueAndFlushWritesEntryToDisk() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(); + FileBasedDictionary dut = CreateFileBasedDictionary(fs, string.Empty); + dut.SetValueAndFlush(TestKey, TestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + [TestCase] + public void SetValueAndFlushUpdatedEntryOnDisk() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(); + FileBasedDictionary dut = CreateFileBasedDictionary(fs, TestEntry); + dut.SetValueAndFlush(TestKey, UpdatedTestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(UpdatedTestEntry); + } + + [TestCase] + [NUnit.Framework.Category(CategoryConstants.ExceptionExpected)] + public void SetValueAndFlushRecoversFromFailedOpenFileStream() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem( + openFileStreamFailurePath: MockEntryFileName + ".tmp", + maxOpenFileStreamFailures: 5, + fileExistsFailurePath: null, + maxFileExistsFailures: 0, + maxMoveAndOverwriteFileFailures: 5); + + FileBasedDictionary dut = CreateFileBasedDictionary(fs, string.Empty); + dut.SetValueAndFlush(TestKey, TestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + [TestCase] + public void SetValueAndFlushRecoversFromDeletedTmp() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem( + openFileStreamFailurePath: null, + maxOpenFileStreamFailures: 0, + fileExistsFailurePath: MockEntryFileName + ".tmp", + maxFileExistsFailures: 5, + maxMoveAndOverwriteFileFailures: 0); + + FileBasedDictionary dut = CreateFileBasedDictionary(fs, string.Empty); + dut.SetValueAndFlush(TestKey, TestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + [TestCase] + [NUnit.Framework.Category(CategoryConstants.ExceptionExpected)] + public void SetValueAndFlushRecoversFromFailedOverwrite() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem( + openFileStreamFailurePath: null, + maxOpenFileStreamFailures: 0, + fileExistsFailurePath: null, + maxFileExistsFailures: 0, + maxMoveAndOverwriteFileFailures: 5); + + FileBasedDictionary dut = CreateFileBasedDictionary(fs, string.Empty); + dut.SetValueAndFlush(TestKey, TestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + [TestCase] + [NUnit.Framework.Category(CategoryConstants.ExceptionExpected)] + public void SetValueAndFlushRecoversFromDeletedTempAndFailedOverwrite() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem( + openFileStreamFailurePath: null, + maxOpenFileStreamFailures: 0, + fileExistsFailurePath: MockEntryFileName + ".tmp", + maxFileExistsFailures: 5, + maxMoveAndOverwriteFileFailures: 5); + + FileBasedDictionary dut = CreateFileBasedDictionary(fs, string.Empty); + dut.SetValueAndFlush(TestKey, TestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + [TestCase] + [NUnit.Framework.Category(CategoryConstants.ExceptionExpected)] + public void SetValueAndFlushRecoversFromMixOfFailures() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(failuresAcrossOpenExistsAndOverwritePath: MockEntryFileName + ".tmp"); + + FileBasedDictionary dut = CreateFileBasedDictionary(fs, string.Empty); + dut.SetValueAndFlush(TestKey, TestValue); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + [TestCase] + public void DeleteFlushesToDisk() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(); + FileBasedDictionary dut = CreateFileBasedDictionary(fs, TestEntry); + dut.RemoveAndFlush(TestKey); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldBeEmpty(); + } + + [TestCase] + public void DeleteUnusedKeyFlushesToDisk() + { + FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(); + FileBasedDictionary dut = CreateFileBasedDictionary(fs, TestEntry); + dut.RemoveAndFlush("UnusedKey"); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry); + } + + private static FileBasedDictionary CreateFileBasedDictionary(FileBasedDictionaryFileSystem fs, string initialContents) + { + fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents)); + + fs.ExpectedOpenFileStreams.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty)); + fs.ExpectedOpenFileStreams.Add(MockEntryFileName, fs.ExpectedFiles[MockEntryFileName]); + + string error; + FileBasedDictionary dut; + FileBasedDictionary.TryCreate(null, MockEntryFileName, fs, out dut, out error).ShouldEqual(true, error); + dut.ShouldNotBeNull(); + + // FileBasedDictionary should only open a file stream to the non-tmp file when being created. At all other times it should + // write to a tmp file and overwrite the non-tmp file + fs.ExpectedOpenFileStreams.Remove(MockEntryFileName); + + return dut; + } + + private class FileBasedDictionaryFileSystem : ConfigurableFileSystem + { + private int openFileStreamFailureCount; + private int maxOpenFileStreamFailures; + private string openFileStreamFailurePath; + + private int fileExistsFailureCount; + private int maxFileExistsFailures; + private string fileExistsFailurePath; + + private int moveAndOverwriteFileFailureCount; + private int maxMoveAndOverwriteFileFailures; + + private string failuresAcrossOpenExistsAndOverwritePath; + private int failuresAcrossOpenExistsAndOverwriteCount; + + public FileBasedDictionaryFileSystem() + { + this.ExpectedOpenFileStreams = new Dictionary(); + } + + public FileBasedDictionaryFileSystem( + string openFileStreamFailurePath, + int maxOpenFileStreamFailures, + string fileExistsFailurePath, + int maxFileExistsFailures, + int maxMoveAndOverwriteFileFailures) + { + this.maxOpenFileStreamFailures = maxOpenFileStreamFailures; + this.openFileStreamFailurePath = openFileStreamFailurePath; + this.fileExistsFailurePath = fileExistsFailurePath; + this.maxFileExistsFailures = maxFileExistsFailures; + this.maxMoveAndOverwriteFileFailures = maxMoveAndOverwriteFileFailures; + this.ExpectedOpenFileStreams = new Dictionary(); + } + + /// + /// Fail a mix of OpenFileStream, FileExists, and Overwrite. + /// + /// + /// Order of failures will be: + /// 1. OpenFileStream + /// 2. FileExists + /// 3. Overwrite + /// + public FileBasedDictionaryFileSystem(string failuresAcrossOpenExistsAndOverwritePath) + { + this.failuresAcrossOpenExistsAndOverwritePath = failuresAcrossOpenExistsAndOverwritePath; + this.ExpectedOpenFileStreams = new Dictionary(); + } + + public Dictionary ExpectedOpenFileStreams { get; } + + public override bool FileExists(string path) + { + if (this.maxFileExistsFailures > 0) + { + if (this.fileExistsFailureCount < this.maxFileExistsFailures && + string.Equals(path, this.fileExistsFailurePath, System.StringComparison.OrdinalIgnoreCase)) + { + if (this.ExpectedFiles.ContainsKey(path)) + { + this.ExpectedFiles.Remove(path); + } + + ++this.fileExistsFailureCount; + } + } + else if (this.failuresAcrossOpenExistsAndOverwritePath != null) + { + if (this.failuresAcrossOpenExistsAndOverwriteCount == 1 && + string.Equals(path, this.failuresAcrossOpenExistsAndOverwritePath, System.StringComparison.OrdinalIgnoreCase)) + { + if (this.ExpectedFiles.ContainsKey(path)) + { + this.ExpectedFiles.Remove(path); + } + + ++this.failuresAcrossOpenExistsAndOverwriteCount; + } + } + + return this.ExpectedFiles.ContainsKey(path); + } + + public override void MoveAndOverwriteFile(string sourceFileName, string destinationFilename) + { + if (this.maxMoveAndOverwriteFileFailures > 0) + { + if (this.moveAndOverwriteFileFailureCount < this.maxMoveAndOverwriteFileFailures) + { + ++this.moveAndOverwriteFileFailureCount; + throw new Win32Exception(); + } + } + else if (this.failuresAcrossOpenExistsAndOverwritePath != null) + { + if (this.failuresAcrossOpenExistsAndOverwriteCount == 2) + { + ++this.failuresAcrossOpenExistsAndOverwriteCount; + throw new Win32Exception(); + } + } + + ReusableMemoryStream source; + this.ExpectedFiles.TryGetValue(sourceFileName, out source).ShouldEqual(true, "Source file does not exist: " + sourceFileName); + this.ExpectedFiles.ContainsKey(destinationFilename).ShouldEqual(true, "MoveAndOverwriteFile expects the destination file to exist: " + destinationFilename); + + this.ExpectedFiles.Remove(sourceFileName); + this.ExpectedFiles[destinationFilename] = source; + } + + public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options) + { + ReusableMemoryStream stream; + this.ExpectedOpenFileStreams.TryGetValue(path, out stream).ShouldEqual(true, "Unexpected access of file: " + path); + + if (this.maxOpenFileStreamFailures > 0) + { + if (this.openFileStreamFailureCount < this.maxOpenFileStreamFailures && + string.Equals(path, this.openFileStreamFailurePath, System.StringComparison.OrdinalIgnoreCase)) + { + ++this.openFileStreamFailureCount; + + if (this.openFileStreamFailureCount % 2 == 0) + { + throw new IOException(); + } + else + { + throw new UnauthorizedAccessException(); + } + } + } + else if (this.failuresAcrossOpenExistsAndOverwritePath != null) + { + if (this.failuresAcrossOpenExistsAndOverwriteCount == 0 && + string.Equals(path, this.failuresAcrossOpenExistsAndOverwritePath, System.StringComparison.OrdinalIgnoreCase)) + { + ++this.failuresAcrossOpenExistsAndOverwriteCount; + throw new IOException(); + } + } + + if (fileMode == FileMode.Create) + { + this.ExpectedFiles[path] = new ReusableMemoryStream(string.Empty); + } + + this.ExpectedFiles.TryGetValue(path, out stream).ShouldEqual(true, "Unexpected access of file: " + path); + return stream; + } + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs b/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs index 7ae9dd857c..99b34dd18f 100644 --- a/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs @@ -7,11 +7,19 @@ namespace GVFS.UnitTests.Common [TestFixture] public class GitVersionTests { + [TestCase] + public void TryParseInstallerName() + { + this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit.exe"); + this.ParseAndValidateInstallerVersion("git-1.2.3.gvfs.4.5.gb16030b-64-bit.exe"); + this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit.EXE"); + } + [TestCase] public void Version_Data_Null_Returns_False() { GitVersion version; - bool success = GitVersion.TryParse(null, out version); + bool success = GitVersion.TryParseVersion(null, out version); success.ShouldEqual(false); } @@ -19,7 +27,7 @@ public void Version_Data_Null_Returns_False() public void Version_Data_Empty_Returns_False() { GitVersion version; - bool success = GitVersion.TryParse(string.Empty, out version); + bool success = GitVersion.TryParseVersion(string.Empty, out version); success.ShouldEqual(false); } @@ -27,7 +35,7 @@ public void Version_Data_Empty_Returns_False() public void Version_Data_Not_Enough_Numbers_Returns_False() { GitVersion version; - bool success = GitVersion.TryParse("2.0.1.test", out version); + bool success = GitVersion.TryParseVersion("2.0.1.test", out version); success.ShouldEqual(false); } @@ -35,7 +43,7 @@ public void Version_Data_Not_Enough_Numbers_Returns_False() public void Version_Data_Too_Many_Numbers_Returns_True() { GitVersion version; - bool success = GitVersion.TryParse("2.0.1.test.1.4.3.6", out version); + bool success = GitVersion.TryParseVersion("2.0.1.test.1.4.3.6", out version); success.ShouldEqual(true); } @@ -43,7 +51,7 @@ public void Version_Data_Too_Many_Numbers_Returns_True() public void Version_Data_Valid_Returns_True() { GitVersion version; - bool success = GitVersion.TryParse("2.0.1.test.1.2", out version); + bool success = GitVersion.TryParseVersion("2.0.1.test.1.2", out version); success.ShouldEqual(true); } @@ -159,13 +167,13 @@ public void Compare_Version_MinorRevision_Greater() public void Allow_Blank_Minor_Revision() { GitVersion version; - GitVersion.TryParse("1.2.3.test.4", out version).ShouldEqual(true); + GitVersion.TryParseVersion("1.2.3.test.4", out version).ShouldEqual(true); version.Major.ShouldEqual(1); version.Minor.ShouldEqual(2); version.Build.ShouldEqual(3); - version.Revision.ShouldEqual(4); version.Platform.ShouldEqual("test"); + version.Revision.ShouldEqual(4); version.MinorRevision.ShouldEqual(0); } @@ -173,14 +181,28 @@ public void Allow_Blank_Minor_Revision() public void Allow_Invalid_Minor_Revision() { GitVersion version; - GitVersion.TryParse("1.2.3.test.4.notint", out version).ShouldEqual(true); + GitVersion.TryParseVersion("1.2.3.test.4.notint", out version).ShouldEqual(true); version.Major.ShouldEqual(1); version.Minor.ShouldEqual(2); version.Build.ShouldEqual(3); - version.Revision.ShouldEqual(4); version.Platform.ShouldEqual("test"); + version.Revision.ShouldEqual(4); version.MinorRevision.ShouldEqual(0); } + + private void ParseAndValidateInstallerVersion(string installerName) + { + GitVersion version; + bool success = GitVersion.TryParseInstallerName(installerName, out version); + success.ShouldBeTrue(); + + version.Major.ShouldEqual(1); + version.Minor.ShouldEqual(2); + version.Build.ShouldEqual(3); + version.Platform.ShouldEqual("gvfs"); + version.Revision.ShouldEqual(4); + version.MinorRevision.ShouldEqual(5); + } } } diff --git a/GVFS/GVFS.UnitTests/Common/JsonEtwTracerTests.cs b/GVFS/GVFS.UnitTests/Common/JsonEtwTracerTests.cs index aca8071ea2..d67b2e09e1 100644 --- a/GVFS/GVFS.UnitTests/Common/JsonEtwTracerTests.cs +++ b/GVFS/GVFS.UnitTests/Common/JsonEtwTracerTests.cs @@ -15,7 +15,7 @@ public class JsonEtwTracerTests [TestCase] public void EventsAreFilteredByVerbosity() { - using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByVerbosity1")) + using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByVerbosity1", useCriticalTelemetryFlag: false)) using (MockListener listener = new MockListener(EventLevel.Informational, Keywords.Any)) { tracer.AddInProcEventListener(listener); @@ -27,7 +27,7 @@ public void EventsAreFilteredByVerbosity() listener.EventNamesRead.ShouldNotContain(name => name.Equals("ShouldNotReceive")); } - using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByVerbosity2")) + using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByVerbosity2", useCriticalTelemetryFlag: false)) using (MockListener listener = new MockListener(EventLevel.Verbose, Keywords.Any)) { tracer.AddInProcEventListener(listener); @@ -44,7 +44,7 @@ public void EventsAreFilteredByVerbosity() public void EventsAreFilteredByKeyword() { // Network filters all but network out - using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByKeyword1")) + using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByKeyword1", useCriticalTelemetryFlag: false)) using (MockListener listener = new MockListener(EventLevel.Verbose, Keywords.Network)) { tracer.AddInProcEventListener(listener); @@ -57,7 +57,7 @@ public void EventsAreFilteredByKeyword() } // Any filters nothing out - using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByKeyword2")) + using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByKeyword2", useCriticalTelemetryFlag: false)) using (MockListener listener = new MockListener(EventLevel.Verbose, Keywords.Any)) { tracer.AddInProcEventListener(listener); @@ -70,7 +70,7 @@ public void EventsAreFilteredByKeyword() } // None filters everything out (including events marked as none) - using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByKeyword3")) + using (JsonEtwTracer tracer = new JsonEtwTracer("Microsoft-GVFS-Test", "EventsAreFilteredByKeyword3", useCriticalTelemetryFlag: false)) using (MockListener listener = new MockListener(EventLevel.Verbose, Keywords.None)) { tracer.AddInProcEventListener(listener); diff --git a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs new file mode 100644 index 0000000000..474e940acc --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs @@ -0,0 +1,146 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock; +using GVFS.UnitTests.Mock.FileSystem; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class PlaceholderDatabaseTests + { + private const string MockEntryFileName = "mock:\\entries.dat"; + + private const string InputGitIgnorePath = ".gitignore"; + private const string InputGitIgnoreSHA = "AE930E4CF715315FC90D4AEC98E16A7398F8BF64"; + + private const string InputGitAttributesPath = ".gitattributes"; + private const string InputGitAttributesSHA = "BB9630E4CF715315FC90D4AEC98E167398F8BF66"; + + private const string InputThirdFilePath = "thirdFile"; + private const string InputThirdFileSHA = "ff9630E00F715315FC90D4AEC98E6A7398F8BF11"; + + private const string ExpectedGitIgnoreEntry = "A " + InputGitIgnorePath + "\0" + InputGitIgnoreSHA + "\r\n"; + private const string ExpectedGitAttributesEntry = "A " + InputGitAttributesPath + "\0" + InputGitAttributesSHA + "\r\n"; + + private const string ExpectedTwoEntries = ExpectedGitIgnoreEntry + ExpectedGitAttributesEntry; + + [TestCase] + public void ParsesExistingDataCorrectly() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + PlaceholderListDatabase dut = CreatePlaceholderListDatabase( + fs, + "A .gitignore\0AE930E4CF715315FC90D4AEC98E16A7398F8BF64\r\n" + + "A Test_EPF_UpdatePlaceholderTests\\LockToPreventDelete\\test.txt\0B6948308A8633CC1ED94285A1F6BF33E35B7C321\r\n" + + "A Test_EPF_UpdatePlaceholderTests\\LockToPreventDelete\\test.txt\0C7048308A8633CC1ED94285A1F6BF33E35B7C321\r\n" + + "A Test_EPF_UpdatePlaceholderTests\\LockToPreventDelete\\test2.txt\0D19198D6EA60F0D66F0432FEC6638D0A73B16E81\r\n" + + "A Test_EPF_UpdatePlaceholderTests\\LockToPreventDelete\\test3.txt\0E45EA0D328E581696CAF1F823686F3665A5F05C1\r\n" + + "A Test_EPF_UpdatePlaceholderTests\\LockToPreventDelete\\test4.txt\0FCB3E2C561649F102DD8110A87DA82F27CC05833\r\n" + + "A Test_EPF_UpdatePlaceholderTests\\LockToPreventUpdate\\test.txt\0E51B377C95076E4C6A9E22A658C5690F324FD0AD\r\n" + + "D Test_EPF_UpdatePlaceholderTests\\LockToPreventUpdate\\test.txt\r\n" + + "D Test_EPF_UpdatePlaceholderTests\\LockToPreventUpdate\\test.txt\r\n" + + "D Test_EPF_UpdatePlaceholderTests\\LockToPreventUpdate\\test.txt\r\n"); + dut.EstimatedCount.ShouldEqual(5); + } + + [TestCase] + public void WritesPlaceholderAddToFile() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, string.Empty); + dut.AddAndFlush(InputGitIgnorePath, InputGitIgnoreSHA); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedGitIgnoreEntry); + + dut.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); + } + + [TestCase] + public void GetAllEntriesReturnsCorrectEntries() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) + { + dut1.AddAndFlush(InputGitIgnorePath, InputGitIgnoreSHA); + dut1.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + dut1.AddAndFlush(InputThirdFilePath, InputThirdFileSHA); + dut1.RemoveAndFlush(InputThirdFilePath); + } + + string error; + PlaceholderListDatabase dut2; + PlaceholderListDatabase.TryCreate(null, MockEntryFileName, fs, out dut2, out error).ShouldEqual(true, error); + List allData = dut2.GetAllEntries(); + allData.Count.ShouldEqual(2); + } + + [TestCase] + public void WriteAllEntriesCorrectlyWritesFile() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + fs.ExpectedFiles.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty)); + + PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, string.Empty); + + List allData = new List() + { + new PlaceholderListDatabase.PlaceholderData(InputGitIgnorePath, InputGitIgnoreSHA), + new PlaceholderListDatabase.PlaceholderData(InputGitAttributesPath, InputGitAttributesSHA) + }; + + dut.WriteAllEntriesAndFlush(allData); + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); + } + + [TestCase] + public void HandlesRaceBetweenAddAndWriteAllEntries() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + fs.ExpectedFiles.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty)); + + PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, ExpectedGitIgnoreEntry); + + List existingEntries = dut.GetAllEntries(); + + dut.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + + dut.WriteAllEntriesAndFlush(existingEntries); + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); + } + + [TestCase] + public void HandlesRaceBetweenRemoveAndWriteAllEntries() + { + const string DeleteGitAttributesEntry = "D .gitattributes\r\n"; + + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + fs.ExpectedFiles.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty)); + + PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, ExpectedTwoEntries); + + List existingEntries = dut.GetAllEntries(); + + dut.RemoveAndFlush(InputGitAttributesPath); + + dut.WriteAllEntriesAndFlush(existingEntries); + fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries + DeleteGitAttributesEntry); + } + + private static PlaceholderListDatabase CreatePlaceholderListDatabase(ConfigurableFileSystem fs, string initialContents) + { + fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents)); + + string error; + PlaceholderListDatabase dut; + PlaceholderListDatabase.TryCreate(null, MockEntryFileName, fs, out dut, out error).ShouldEqual(true, error); + dut.ShouldNotBeNull(); + return dut; + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/RetryConfigTests.cs b/GVFS/GVFS.UnitTests/Common/RetryConfigTests.cs index 0f8c45b26d..add8718428 100644 --- a/GVFS/GVFS.UnitTests/Common/RetryConfigTests.cs +++ b/GVFS/GVFS.UnitTests/Common/RetryConfigTests.cs @@ -2,6 +2,7 @@ using GVFS.Common.Git; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Mock.Git; using NUnit.Framework; using System; @@ -16,7 +17,7 @@ public class RetryConfigTests public void TryLoadConfigFailsWhenGitFailsToReadConfig() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.max-retries", () => new GitProcess.Result(string.Empty, ReadConfigFailureMessage, GitProcess.Result.GenericFailureCode)); gitProcess.SetExpectedCommandResult("config gvfs.timeout-seconds", () => new GitProcess.Result(string.Empty, ReadConfigFailureMessage, GitProcess.Result.GenericFailureCode)); @@ -30,7 +31,7 @@ public void TryLoadConfigFailsWhenGitFailsToReadConfig() public void TryLoadConfigUsesDefaultValuesWhenEntriesNotInConfig() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.max-retries", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.GenericFailureCode)); gitProcess.SetExpectedCommandResult("config gvfs.timeout-seconds", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.GenericFailureCode)); @@ -47,7 +48,7 @@ public void TryLoadConfigUsesDefaultValuesWhenEntriesNotInConfig() public void TryLoadConfigUsesDefaultValuesWhenEntriesAreBlank() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.max-retries", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); gitProcess.SetExpectedCommandResult("config gvfs.timeout-seconds", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.SuccessCode)); @@ -64,7 +65,7 @@ public void TryLoadConfigUsesDefaultValuesWhenEntriesAreBlank() public void TryLoadConfigEnforcesMinimumValuesOnMaxRetries() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.max-retries", () => new GitProcess.Result("-1", string.Empty, GitProcess.Result.SuccessCode)); gitProcess.SetExpectedCommandResult("config gvfs.timeout-seconds", () => new GitProcess.Result("30", string.Empty, GitProcess.Result.SuccessCode)); @@ -78,7 +79,7 @@ public void TryLoadConfigEnforcesMinimumValuesOnMaxRetries() public void TryLoadConfigEnforcesMinimumValuesOnTimeout() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.max-retries", () => new GitProcess.Result("3", string.Empty, GitProcess.Result.SuccessCode)); gitProcess.SetExpectedCommandResult("config gvfs.timeout-seconds", () => new GitProcess.Result("-1", string.Empty, GitProcess.Result.SuccessCode)); @@ -95,7 +96,7 @@ public void TryLoadConfigUsesConfiguredValues() int timeoutSeconds = RetryConfig.DefaultTimeoutSeconds + 1; MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.max-retries", () => new GitProcess.Result(maxRetries.ToString(), string.Empty, GitProcess.Result.SuccessCode)); gitProcess.SetExpectedCommandResult("config gvfs.timeout-seconds", () => new GitProcess.Result(timeoutSeconds.ToString(), string.Empty, GitProcess.Result.SuccessCode)); diff --git a/GVFS/GVFS.UnitTests/Common/RetryWrapperTests.cs b/GVFS/GVFS.UnitTests/Common/RetryWrapperTests.cs index 1e1b46ecca..eaaf270f76 100644 --- a/GVFS/GVFS.UnitTests/Common/RetryWrapperTests.cs +++ b/GVFS/GVFS.UnitTests/Common/RetryWrapperTests.cs @@ -4,6 +4,7 @@ using NUnit.Framework; using System; using System.IO; +using System.Threading; using System.Threading.Tasks; namespace GVFS.UnitTests.Common @@ -17,7 +18,7 @@ public void WillRetryOnIOException() { const int ExpectedTries = 5; - RetryWrapper dut = new RetryWrapper(ExpectedTries, exponentialBackoffBase: 0); + RetryWrapper dut = new RetryWrapper(ExpectedTries, new CancellationToken(canceled: false), exponentialBackoffBase: 0); int actualTries = 0; RetryWrapper.InvocationResult output = dut.Invoke( @@ -37,7 +38,7 @@ public void WillNotRetryForGenericExceptions() { const int MaxTries = 5; - RetryWrapper dut = new RetryWrapper(MaxTries, exponentialBackoffBase: 0); + RetryWrapper dut = new RetryWrapper(MaxTries, new CancellationToken(canceled: false), exponentialBackoffBase: 0); Assert.Throws( () => @@ -46,6 +47,93 @@ public void WillNotRetryForGenericExceptions() }); } + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void WillNotMakeAnyAttemptWhenInitiallyCanceled() + { + const int MaxTries = 5; + int actualTries = 0; + + RetryWrapper dut = new RetryWrapper(MaxTries, new CancellationToken(canceled: true), exponentialBackoffBase: 0); + + Assert.Throws( + () => + { + RetryWrapper.InvocationResult output = dut.Invoke(tryCount => + { + ++actualTries; + return new RetryWrapper.CallbackResult(true); + }); + }); + + actualTries.ShouldEqual(0); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void WillNotRetryForWhenCanceledDuringAttempts() + { + const int MaxTries = 5; + int actualTries = 0; + int expectedTries = 3; + + using (CancellationTokenSource tokenSource = new CancellationTokenSource()) + { + RetryWrapper dut = new RetryWrapper(MaxTries, tokenSource.Token, exponentialBackoffBase: 0); + + Assert.Throws( + () => + { + RetryWrapper.InvocationResult output = dut.Invoke(tryCount => + { + ++actualTries; + + if (actualTries == expectedTries) + { + tokenSource.Cancel(); + } + + return new RetryWrapper.CallbackResult(new Exception("Test"), shouldRetry: true); + }); + }); + + actualTries.ShouldEqual(expectedTries); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void WillNotRetryWhenCancelledDuringBackoff() + { + const int MaxTries = 5; + int actualTries = 0; + int expectedTries = 2; // 2 because RetryWrapper does not wait after the first failure + + using (CancellationTokenSource tokenSource = new CancellationTokenSource()) + { + RetryWrapper dut = new RetryWrapper(MaxTries, tokenSource.Token, exponentialBackoffBase: 300); + + Task.Run(() => + { + // Wait 3 seconds and cancel + Thread.Sleep(1000 * 3); + tokenSource.Cancel(); + }); + + Assert.Throws( + () => + { + RetryWrapper.InvocationResult output = dut.Invoke(tryCount => + { + ++actualTries; + return new RetryWrapper.CallbackResult(new Exception("Test"), shouldRetry: true); + }); + }); + + actualTries.ShouldEqual(expectedTries); + } + } + [TestCase] [Category(CategoryConstants.ExceptionExpected)] public void OnFailureIsCalledWhenEventHandlerAttached() @@ -53,7 +141,7 @@ public void OnFailureIsCalledWhenEventHandlerAttached() const int MaxTries = 5; const int ExpectedFailures = 5; - RetryWrapper dut = new RetryWrapper(MaxTries, exponentialBackoffBase: 0); + RetryWrapper dut = new RetryWrapper(MaxTries, new CancellationToken(canceled: false), exponentialBackoffBase: 0); int actualFailures = 0; dut.OnFailure += errorArgs => actualFailures++; @@ -75,7 +163,7 @@ public void OnSuccessIsOnlyCalledOnce() const int ExpectedFailures = 0; const int ExpectedTries = 1; - RetryWrapper dut = new RetryWrapper(MaxTries, exponentialBackoffBase: 0); + RetryWrapper dut = new RetryWrapper(MaxTries, new CancellationToken(canceled: false), exponentialBackoffBase: 0); int actualFailures = 0; dut.OnFailure += errorArgs => actualFailures++; @@ -101,7 +189,7 @@ public void WillNotRetryWhenNotRequested() const int ExpectedFailures = 1; const int ExpectedTries = 1; - RetryWrapper dut = new RetryWrapper(MaxTries, exponentialBackoffBase: 0); + RetryWrapper dut = new RetryWrapper(MaxTries, new CancellationToken(canceled: false), exponentialBackoffBase: 0); int actualFailures = 0; dut.OnFailure += errorArgs => actualFailures++; @@ -111,7 +199,7 @@ public void WillNotRetryWhenNotRequested() tryCount => { actualTries++; - return new RetryWrapper.CallbackResult(new Exception("Test"), false); + return new RetryWrapper.CallbackResult(new Exception("Test"), shouldRetry: false); }); output.Succeeded.ShouldEqual(false); @@ -127,7 +215,7 @@ public void WillRetryWhenRequested() const int ExpectedFailures = 5; const int ExpectedTries = 5; - RetryWrapper dut = new RetryWrapper(MaxTries, exponentialBackoffBase: 0); + RetryWrapper dut = new RetryWrapper(MaxTries, new CancellationToken(canceled: false), exponentialBackoffBase: 0); int actualFailures = 0; dut.OnFailure += errorArgs => actualFailures++; @@ -137,7 +225,7 @@ public void WillRetryWhenRequested() tryCount => { actualTries++; - return new RetryWrapper.CallbackResult(new Exception("Test"), true); + return new RetryWrapper.CallbackResult(new Exception("Test"), shouldRetry: true); }); output.Succeeded.ShouldEqual(false); diff --git a/GVFS/GVFS.UnitTests/FastFetch/BatchObjectDownloadJobTests.cs b/GVFS/GVFS.UnitTests/FastFetch/BatchObjectDownloadJobTests.cs index 670e3ecf2e..3becf4717b 100644 --- a/GVFS/GVFS.UnitTests/FastFetch/BatchObjectDownloadJobTests.cs +++ b/GVFS/GVFS.UnitTests/FastFetch/BatchObjectDownloadJobTests.cs @@ -61,7 +61,7 @@ public void OnlyRequestsObjectsNotDownloaded() tracer, enlistment, httpObjects, - new MockPhysicalGitObjects(tracer, enlistment, httpObjects)); + new MockPhysicalGitObjects(tracer, null, enlistment, httpObjects)); dut.Start(); dut.WaitForCompletion(); diff --git a/GVFS/GVFS.UnitTests/FastFetch/DiffHelperTests.cs b/GVFS/GVFS.UnitTests/FastFetch/DiffHelperTests.cs index 2d78452109..b35eaef6c0 100644 --- a/GVFS/GVFS.UnitTests/FastFetch/DiffHelperTests.cs +++ b/GVFS/GVFS.UnitTests/FastFetch/DiffHelperTests.cs @@ -1,6 +1,8 @@ -using GVFS.Common.Git; +using FastFetch.Git; +using GVFS.Common.Git; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Mock.Git; using NUnit.Framework; using System.Collections.Generic; @@ -43,7 +45,7 @@ public class DiffHelperTests public void CanParseDiffForwards() { MockTracer tracer = new MockTracer(); - DiffHelper diffForwards = new DiffHelper(tracer, new MockEnlistment(), new List()); + DiffHelper diffForwards = new DiffHelper(tracer, new MockEnlistment(), new List(), new List()); diffForwards.ParseDiffFile(GetDataPath("forward.txt"), "xx:\\fakeRepo"); // File added, file edited, file renamed, folder => file, edit-rename file @@ -69,7 +71,7 @@ public void CanParseDiffForwards() public void CanParseBackwardsDiff() { MockTracer tracer = new MockTracer(); - DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), new List(), new List()); diffBackwards.ParseDiffFile(GetDataPath("backward.txt"), "xx:\\fakeRepo"); // File > folder, deleted file, edited file, renamed file, rename-edit file @@ -93,7 +95,7 @@ public void CanParseBackwardsDiff() public void ParsesCaseChangesAsAdds() { MockTracer tracer = new MockTracer(); - DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), new List(), new List()); diffBackwards.ParseDiffFile(GetDataPath("caseChange.txt"), "xx:\\fakeRepo"); diffBackwards.RequiredBlobs.Count.ShouldEqual(2); @@ -110,10 +112,10 @@ public void ParsesCaseChangesAsAdds() public void DetectsFailuresInDiffTree() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("diff-tree -r -t sha1 sha2", () => new GitProcess.Result(string.Empty, string.Empty, 1)); - DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), gitProcess, new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), gitProcess, new List(), new List()); diffBackwards.PerformDiff("sha1", "sha2"); diffBackwards.HasFailures.ShouldEqual(true); } @@ -122,10 +124,10 @@ public void DetectsFailuresInDiffTree() public void DetectsFailuresInLsTree() { MockTracer tracer = new MockTracer(); - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("ls-tree -r -t sha1", () => new GitProcess.Result(string.Empty, string.Empty, 1)); - DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), gitProcess, new List()); + DiffHelper diffBackwards = new DiffHelper(tracer, new MockEnlistment(), gitProcess, new List(), new List()); diffBackwards.PerformDiff(null, "sha1"); diffBackwards.HasFailures.ShouldEqual(true); } diff --git a/GVFS/GVFS.UnitTests/FastFetch/FastFetchTracingTests.cs b/GVFS/GVFS.UnitTests/FastFetch/FastFetchTracingTests.cs index c18c7da497..a717b3adc0 100644 --- a/GVFS/GVFS.UnitTests/FastFetch/FastFetchTracingTests.cs +++ b/GVFS/GVFS.UnitTests/FastFetch/FastFetchTracingTests.cs @@ -22,7 +22,7 @@ public void ErrorsForBatchObjectDownloadJob() { MockEnlistment enlistment = new MockEnlistment(); MockHttpGitObjects httpGitObjects = new MockHttpGitObjects(tracer, enlistment); - MockPhysicalGitObjects gitObjects = new MockPhysicalGitObjects(tracer, enlistment, httpGitObjects); + MockPhysicalGitObjects gitObjects = new MockPhysicalGitObjects(tracer, null, enlistment, httpGitObjects); BlockingCollection input = new BlockingCollection(); input.Add(FakeSha); @@ -48,7 +48,7 @@ public void SuccessForBatchObjectDownloadJob() MockEnlistment enlistment = new MockEnlistment(); MockHttpGitObjects httpGitObjects = new MockHttpGitObjects(tracer, enlistment); httpGitObjects.AddBlobContent(FakeSha, FakeShaContents); - MockPhysicalGitObjects gitObjects = new MockPhysicalGitObjects(tracer, enlistment, httpGitObjects); + MockPhysicalGitObjects gitObjects = new MockPhysicalGitObjects(tracer, null, enlistment, httpGitObjects); BlockingCollection input = new BlockingCollection(); input.Add(FakeSha); @@ -74,7 +74,7 @@ public void ErrorsForIndexPackFile() using (JsonEtwTracer tracer = CreateTracer()) { MockEnlistment enlistment = new MockEnlistment(); - MockPhysicalGitObjects gitObjects = new MockPhysicalGitObjects(tracer, enlistment, null); + MockPhysicalGitObjects gitObjects = new MockPhysicalGitObjects(tracer, null, enlistment, null); BlockingCollection input = new BlockingCollection(); BlobDownloadRequest downloadRequest = new BlobDownloadRequest(new string[] { FakeSha }); @@ -89,7 +89,7 @@ public void ErrorsForIndexPackFile() private static JsonEtwTracer CreateTracer() { - return new JsonEtwTracer("Microsoft-GVFS-Test", "FastFetchTest"); + return new JsonEtwTracer("Microsoft-GVFS-Test", "FastFetchTest", useCriticalTelemetryFlag: false); } } } diff --git a/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj b/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj index 895bac8825..9f4609f3ea 100644 --- a/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj +++ b/GVFS/GVFS.UnitTests/GVFS.UnitTests.csproj @@ -40,6 +40,21 @@ true + + False + ..\..\..\packages\Microsoft.Database.Collections.Generic.1.9.4\lib\net40\Esent.Collections.dll + True + + + False + ..\..\..\packages\ManagedEsent.1.9.4\lib\net40\Esent.Interop.dll + True + + + False + ..\..\..\packages\Microsoft.Database.Isam.1.9.4\lib\net40\Esent.Isam.dll + True + False ..\..\..\packages\Microsoft.Diagnostics.Tracing.EventSource.Redist.1.1.28\lib\net40\Microsoft.Diagnostics.Tracing.EventSource.dll @@ -66,8 +81,11 @@ - + + + + @@ -89,18 +107,22 @@ - - + + + + + + @@ -130,6 +152,10 @@ {374bf1e5-0b2d-4d4a-bd5e-4212299def09} GVFS.Common + + {fb0831ae-9997-401b-b31f-3a065fdbeb20} + GvLib.Managed + {1118b427-7063-422f-83b9-5023c8ec5a7a} GVFS.GVFlt diff --git a/GVFS/GVFS.UnitTests/GVFlt/DotGit/AlwaysExcludeFileTests.cs b/GVFS/GVFS.UnitTests/GVFlt/DotGit/AlwaysExcludeFileTests.cs index cf46060f1b..9aa11a4c76 100644 --- a/GVFS/GVFS.UnitTests/GVFlt/DotGit/AlwaysExcludeFileTests.cs +++ b/GVFS/GVFS.UnitTests/GVFlt/DotGit/AlwaysExcludeFileTests.cs @@ -24,7 +24,7 @@ public void HasDefaultEntriesAfterLoad() } [TestCase] - public void WritesOnFolderChange() + public void WritesParentFoldersWithoutDuplicates() { string alwaysExcludeFilePath = Path.Combine(this.Repo.GitParentPath, GVFS.Common.GVFSConstants.DotGit.Info.AlwaysExcludeName); AlwaysExcludeFile alwaysExcludeFile = new AlwaysExcludeFile(this.Repo.Context, alwaysExcludeFilePath); @@ -32,17 +32,18 @@ public void WritesOnFolderChange() alwaysExcludeFile.LoadOrCreate(); this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\B\\C", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\D\\E", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\C\\E.txt", isFolder: false); + alwaysExcludeFile.AddEntriesForFile("a\\1.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\2.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\3.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\b\\1.txt"); + alwaysExcludeFile.AddEntriesForFile("c\\1.txt"); - List expectedContents = new List() { "*", "!/A", "!/A/B", "!/A/B/C", "!/A/B/C/*", "!/A/D", "!/A/D/E", "!/A/D/E/*", "!/A/C", "!/A/C/*" }; + List expectedContents = new List() { "*", "!/a/", "!/a/1.txt", "!/a/2.txt", "!/a/3.txt", "!/a/b/", "!/a/b/1.txt", "!/c/", "!/c/1.txt" }; this.CheckFileContents(alwaysExcludeFilePath, expectedContents); } [TestCase] - - public void DoesNotWriteDuplicateFolderEntries() + public void HandlesCaseCorrectly() { string alwaysExcludeFilePath = Path.Combine(this.Repo.GitParentPath, GVFS.Common.GVFSConstants.DotGit.Info.AlwaysExcludeName); AlwaysExcludeFile alwaysExcludeFile = new AlwaysExcludeFile(this.Repo.Context, alwaysExcludeFilePath); @@ -50,16 +51,14 @@ public void DoesNotWriteDuplicateFolderEntries() alwaysExcludeFile.LoadOrCreate(); this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\B", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("a\\b", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("a\\b.txt", isFolder: false); - alwaysExcludeFile.AddEntriesForFileOrFolder("a\\b\\c.txt", isFolder: false); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\D", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\d", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("a\\f", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("a\\F", isFolder: true); + alwaysExcludeFile.AddEntriesForFile("a\\1.txt"); + alwaysExcludeFile.AddEntriesForFile("A\\2.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\b\\1.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\B\\2.txt"); + alwaysExcludeFile.AddEntriesForFile("A\\b\\3.txt"); + alwaysExcludeFile.AddEntriesForFile("A\\B\\4.txt"); - List expectedContents = new List() { "*", "!/A", "!/A/B", "!/A/B/*", "!/a/*", "!/A/D", "!/A/D/*", "!/a/f", "!/a/f/*" }; + List expectedContents = new List() { "*", "!/a/", "!/a/1.txt", "!/A/2.txt", "!/a/b/", "!/a/b/1.txt", "!/a/B/2.txt", "!/A/b/3.txt", "!/A/B/4.txt" }; this.CheckFileContents(alwaysExcludeFilePath, expectedContents); } @@ -72,17 +71,54 @@ public void WritesAfterLoad() alwaysExcludeFile.LoadOrCreate(); this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\B", isFolder: true); - alwaysExcludeFile.AddEntriesForFileOrFolder("A\\D", isFolder: true); + alwaysExcludeFile.AddEntriesForFile("a\\1.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\2.txt"); - List expectedContents = new List() { "*", "!/A", "!/A/B", "!/A/B/*", "!/A/D", "!/A/D/*" }; + List expectedContents = new List() { "*", "!/a/", "!/a/1.txt", "!/a/2.txt" }; this.CheckFileContents(alwaysExcludeFilePath, expectedContents); alwaysExcludeFile = new AlwaysExcludeFile(this.Repo.Context, alwaysExcludeFilePath); alwaysExcludeFile.LoadOrCreate(); - alwaysExcludeFile.AddEntriesForFileOrFolder("a\\f", isFolder: true); + alwaysExcludeFile.AddEntriesForFile("a\\3.txt"); + + expectedContents = new List() { "*", "!/a/", "!/a/1.txt", "!/a/2.txt", "!/a/3.txt" }; + this.CheckFileContents(alwaysExcludeFilePath, expectedContents); + } + + [TestCase] + public void RemovesEntries() + { + string alwaysExcludeFilePath = Path.Combine(this.Repo.GitParentPath, GVFS.Common.GVFSConstants.DotGit.Info.AlwaysExcludeName); + AlwaysExcludeFile alwaysExcludeFile = new AlwaysExcludeFile(this.Repo.Context, alwaysExcludeFilePath); + this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(false); + alwaysExcludeFile.LoadOrCreate(); + this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(true); + + alwaysExcludeFile.AddEntriesForFile("a\\1.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\2.txt"); + alwaysExcludeFile.RemoveEntriesForFiles(new List { "a\\1.txt" }); + alwaysExcludeFile.FlushAndClose(); + + List expectedContents = new List() { "*", "!/a/", "!/a/2.txt" }; + this.CheckFileContents(alwaysExcludeFilePath, expectedContents); + } + + [TestCase] + public void RemovesEntriesWithDifferentCase() + { + string alwaysExcludeFilePath = Path.Combine(this.Repo.GitParentPath, GVFS.Common.GVFSConstants.DotGit.Info.AlwaysExcludeName); + AlwaysExcludeFile alwaysExcludeFile = new AlwaysExcludeFile(this.Repo.Context, alwaysExcludeFilePath); + this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(false); + alwaysExcludeFile.LoadOrCreate(); + this.Repo.Context.FileSystem.FileExists(alwaysExcludeFilePath).ShouldEqual(true); + + alwaysExcludeFile.AddEntriesForFile("a\\x.txt"); + alwaysExcludeFile.AddEntriesForFile("A\\y.txt"); + alwaysExcludeFile.AddEntriesForFile("a\\Z.txt"); + alwaysExcludeFile.RemoveEntriesForFiles(new List { "a\\y.txt", "a\\z.txt" }); + alwaysExcludeFile.FlushAndClose(); - expectedContents = new List() { "*", "!/A", "!/A/B", "!/A/B/*", "!/A/D", "!/A/D/*", "!/a/f", "!/a/f/*" }; + List expectedContents = new List() { "*", "!/a/", "!/a/x.txt" }; this.CheckFileContents(alwaysExcludeFilePath, expectedContents); } diff --git a/GVFS/GVFS.UnitTests/GVFlt/GVFltCallbacksTests.cs b/GVFS/GVFS.UnitTests/GVFlt/GVFltCallbacksTests.cs index f12c2ba717..879eea28fd 100644 --- a/GVFS/GVFS.UnitTests/GVFlt/GVFltCallbacksTests.cs +++ b/GVFS/GVFS.UnitTests/GVFlt/GVFltCallbacksTests.cs @@ -1,10 +1,21 @@ -using GVFS.GVFlt; +using GVFS.Common; +using GVFS.GVFlt; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.Git; +using GVFS.UnitTests.Mock.GvFlt; +using GVFS.UnitTests.Mock.GVFS.GvFlt; +using GVFS.UnitTests.Mock.GVFS.GvFlt.DotGit; +using GVFS.UnitTests.Virtual; +using GvLib; using NUnit.Framework; +using System; +using System.Threading.Tasks; namespace GVFS.UnitTests.GVFlt.DotGit { - public class GVFltCallbacksTests + public class GVFltCallbacksTests : TestsWithCommonRepo { [TestCase] public void CannotDeleteIndexOrPacks() @@ -43,5 +54,464 @@ public void IsPathMonitoredForWrites() GVFltCallbacks.IsPathMonitoredForWrites(@".git\objects\pack").ShouldEqual(false); GVFltCallbacks.IsPathMonitoredForWrites(@".git\objects").ShouldEqual(false); } + + [TestCase] + public void OnStartDirectoryEnumerationReturnsPendingWhenResultsNotInMemory() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + gitIndexProjection.EnumerationInMemory = false; + mockGvFlt.OnStartDirectoryEnumeration(1, enumerationGuid, "test").ShouldEqual(NtStatus.Pending); + mockGvFlt.WaitForCompletionStatus().ShouldEqual(NtStatus.Success); + mockGvFlt.OnEndDirectoryEnumeration(enumerationGuid).ShouldEqual(NtStatus.Success); + } + } + + [TestCase] + public void OnStartDirectoryEnumerationReturnsSuccessWhenResultsInMemory() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + gitIndexProjection.EnumerationInMemory = true; + mockGvFlt.OnStartDirectoryEnumeration(1, enumerationGuid, "test").ShouldEqual(NtStatus.Success); + mockGvFlt.OnEndDirectoryEnumeration(enumerationGuid).ShouldEqual(NtStatus.Success); + } + } + + [TestCase] + public void GetPlaceholderInformationHandlerPathNotProjected() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + mockGvFlt.OnGetPlaceholderInformation(1, "doesNotExist", 0, 0, 0, 0, 1, "UnitTests").ShouldEqual(NtStatus.ObjectNameNotFound); + } + } + + [TestCase] + public void GetPlaceholderInformationHandlerPathProjected() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + mockGvFlt.OnGetPlaceholderInformation(1, "test.txt", 0, 0, 0, 0, 1, "UnitTests").ShouldEqual(NtStatus.Pending); + mockGvFlt.WaitForCompletionStatus().ShouldEqual(NtStatus.Success); + mockGvFlt.CreatedPlaceholders.ShouldContain(entry => entry == "test.txt"); + gitIndexProjection.PlaceholdersCreated.ShouldContain(entry => entry == "test.txt"); + } + } + + [TestCase] + public void GetPlaceholderInformationHandlerCancelledBeforeSchedulingAsync() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + gitIndexProjection.BlockIsPathProjected(willWaitForRequest: true); + + Task.Run(() => + { + // Wait for OnGetPlaceholderInformation to call IsPathProjected and then while it's blocked there + // call OnCancelCommand + gitIndexProjection.WaitForIsPathProjected(); + mockGvFlt.OnCancelCommand(1); + gitIndexProjection.UnblockIsPathProjected(); + }); + + mockGvFlt.OnGetPlaceholderInformation(1, "test.txt", 0, 0, 0, 0, 1, "UnitTests").ShouldEqual(NtStatus.Pending); + + // Cancelling before GetPlaceholderInformation has registered the command results in placeholders being created + mockGvFlt.WaitForPlaceholderCreate(); + gitIndexProjection.WaitForPlaceholderCreate(); + mockGvFlt.CreatedPlaceholders.ShouldContain(entry => entry == "test.txt"); + gitIndexProjection.PlaceholdersCreated.ShouldContain(entry => entry == "test.txt"); + } + } + + [TestCase] + public void GetPlaceholderInformationHandlerCancelledDuringAsyncCallback() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + gitIndexProjection.BlockGetProjectedFileInfo(willWaitForRequest: true); + mockGvFlt.OnGetPlaceholderInformation(1, "test.txt", 0, 0, 0, 0, 1, "UnitTests").ShouldEqual(NtStatus.Pending); + gitIndexProjection.WaitForGetProjectedFileInfo(); + mockGvFlt.OnCancelCommand(1); + gitIndexProjection.UnblockGetProjectedFileInfo(); + + // Cancelling in the middle of GetPlaceholderInformation still allows it to create placeholders when the cancellation does not + // interrupt network requests + mockGvFlt.WaitForPlaceholderCreate(); + gitIndexProjection.WaitForPlaceholderCreate(); + mockGvFlt.CreatedPlaceholders.ShouldContain(entry => entry == "test.txt"); + gitIndexProjection.PlaceholdersCreated.ShouldContain(entry => entry == "test.txt"); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void GetPlaceholderInformationHandlerCancelledDuringNetworkRequest() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.WaitRelatedEventName = "GVFltGetPlaceholderInformationAsyncHandler_GetProjectedGVFltFileInfoAndShaCancelled"; + gitIndexProjection.ThrowOperationCanceledExceptionOnProjectionRequest = true; + mockGvFlt.OnGetPlaceholderInformation(1, "test.txt", 0, 0, 0, 0, 1, "UnitTests").ShouldEqual(NtStatus.Pending); + + // Cancelling in the middle of GetPlaceholderInformation in the middle of a network request should not result in placeholder + // getting created + mockTracker.WaitForRelatedEvent(); + mockGvFlt.CreatedPlaceholders.ShouldNotContain(entry => entry == "test.txt"); + gitIndexProjection.PlaceholdersCreated.ShouldNotContain(entry => entry == "test.txt"); + } + } + + [TestCase] + public void OnGetFileStreamReturnsInvalidParameterWhenOffsetNonZero() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + + byte[] contentId = GVFltCallbacks.ConvertShaToContentId("0123456789012345678901234567890123456789"); + byte[] epochId = GVFltCallbacks.GetEpochId(); + + mockGvFlt.OnGetFileStream( + commandId: 1, + relativePath: "test.txt", + byteOffset: 10, + length: 100, + streamGuid: Guid.NewGuid(), + contentId: contentId, + epochId: epochId, + triggeringProcessId: 2, + triggeringProcessImageFileName: "UnitTest").ShouldEqual(NtStatus.InvalidParameter); + } + } + + [TestCase] + public void OnGetFileStreamReturnsInternalErrorWhenPlaceholderVersionDoesNotMatchExpected() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + + byte[] contentId = GVFltCallbacks.ConvertShaToContentId("0123456789012345678901234567890123456789"); + byte[] epochId = new byte[] { GVFltCallbacks.PlaceholderVersion + 1 }; + + mockGvFlt.OnGetFileStream( + commandId: 1, + relativePath: "test.txt", + byteOffset: 0, + length: 100, + streamGuid: Guid.NewGuid(), + contentId: contentId, + epochId: epochId, + triggeringProcessId: 2, + triggeringProcessImageFileName: "UnitTest").ShouldEqual(NtStatus.InternalError); + } + } + + [TestCase] + public void OnGetFileStreamReturnsPendingAndCompletesWithSuccessWhenNoFailures() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + + byte[] contentId = GVFltCallbacks.ConvertShaToContentId("0123456789012345678901234567890123456789"); + byte[] epochId = GVFltCallbacks.GetEpochId(); + + uint fileLength = 100; + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.FileLength = fileLength; + mockGvFlt.WriteFileReturnStatus = NtStatus.Success; + + mockGvFlt.OnGetFileStream( + commandId: 1, + relativePath: "test.txt", + byteOffset: 0, + length: fileLength, + streamGuid: Guid.NewGuid(), + contentId: contentId, + epochId: epochId, + triggeringProcessId: 2, + triggeringProcessImageFileName: "UnitTest").ShouldEqual(NtStatus.Pending); + + mockGvFlt.WaitForCompletionStatus().ShouldEqual(NtStatus.Success); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void OnGetFileStreamHandlesTryCopyBlobContentStreamThrowingOperationCanceled() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + + byte[] contentId = GVFltCallbacks.ConvertShaToContentId("0123456789012345678901234567890123456789"); + byte[] epochId = GVFltCallbacks.GetEpochId(); + + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.WaitRelatedEventName = "GVFltGetFileStreamHandlerAsyncHandler_OperationCancelled"; + mockGVFSGitObjects.CancelTryCopyBlobContentStream = true; + + mockGvFlt.OnGetFileStream( + commandId: 1, + relativePath: "test.txt", + byteOffset: 0, + length: 100, + streamGuid: Guid.NewGuid(), + contentId: contentId, + epochId: epochId, + triggeringProcessId: 2, + triggeringProcessImageFileName: "UnitTest").ShouldEqual(NtStatus.Pending); + + mockTracker.WaitForRelatedEvent(); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void OnGetFileStreamHandlesCancellationDuringWriteAction() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + + byte[] contentId = GVFltCallbacks.ConvertShaToContentId("0123456789012345678901234567890123456789"); + byte[] epochId = GVFltCallbacks.GetEpochId(); + + uint fileLength = 100; + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.FileLength = fileLength; + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.WaitRelatedEventName = "GVFltGetFileStreamHandlerAsyncHandler_OperationCancelled"; + + mockGvFlt.BlockCreateWriteBuffer(willWaitForRequest: true); + mockGvFlt.OnGetFileStream( + commandId: 1, + relativePath: "test.txt", + byteOffset: 0, + length: fileLength, + streamGuid: Guid.NewGuid(), + contentId: contentId, + epochId: epochId, + triggeringProcessId: 2, + triggeringProcessImageFileName: "UnitTest").ShouldEqual(NtStatus.Pending); + + mockGvFlt.WaitForCreateWriteBuffer(); + mockGvFlt.OnCancelCommand(1); + mockGvFlt.UnblockCreateWriteBuffer(); + mockTracker.WaitForRelatedEvent(); + } + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void OnGetFileStreamHandlesGvWriteFailure() + { + using (MockVirtualizationInstance mockGvFlt = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + { + GVFltCallbacks callbacks = new GVFltCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + blobSizes: null, + gvflt: mockGvFlt, + gitIndexProjection: gitIndexProjection, + reliableBackgroundOperations: new MockReliableBackgroundOperations()); + + string error; + callbacks.TryStart(out error).ShouldEqual(true); + + Guid enumerationGuid = Guid.NewGuid(); + + byte[] contentId = GVFltCallbacks.ConvertShaToContentId("0123456789012345678901234567890123456789"); + byte[] epochId = GVFltCallbacks.GetEpochId(); + + uint fileLength = 100; + MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; + mockGVFSGitObjects.FileLength = fileLength; + + MockTracer mockTracker = this.Repo.Context.Tracer as MockTracer; + mockTracker.WaitRelatedEventName = "GVFltGetFileStreamHandlerAsyncHandler_OperationCancelled"; + + mockGvFlt.WriteFileReturnStatus = NtStatus.InternalError; + mockGvFlt.OnGetFileStream( + commandId: 1, + relativePath: "test.txt", + byteOffset: 0, + length: fileLength, + streamGuid: Guid.NewGuid(), + contentId: contentId, + epochId: epochId, + triggeringProcessId: 2, + triggeringProcessImageFileName: "UnitTest").ShouldEqual(NtStatus.Pending); + + mockGvFlt.WaitForCompletionStatus().ShouldEqual(mockGvFlt.WriteFileReturnStatus); + } + } } } \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs b/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs index 2be5f37422..426750e18a 100644 --- a/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GVFSGitObjectsTests.cs @@ -1,16 +1,20 @@ using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Tests.Should; using GVFS.UnitTests.Category; using GVFS.UnitTests.Mock; using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; +using GVFS.UnitTests.Mock.Git; using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; using System.Net; using System.Reflection; +using System.Threading; namespace GVFS.UnitTests.Git { @@ -18,33 +22,49 @@ namespace GVFS.UnitTests.Git public class GVFSGitObjectsTests { private const string ValidTestObjectFileContents = "421dc4df5e1de427e363b8acd9ddb2d41385dbdf"; - private string tempFolder; + private const string TestEnlistmentRoot = "mock:\\src"; - [OneTimeSetUp] - public void Setup() + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void CatchesFileNotFoundAfterFileDeleted() { - this.tempFolder = Path.Combine(Environment.CurrentDirectory, Path.GetRandomFileName()); - string packFolder = Path.Combine(this.tempFolder, ".gvfs\\gitObjectCache\\pack"); - Directory.CreateDirectory(packFolder); - } + MockFileSystemWithCallbacks fileSystem = new MockFileSystemWithCallbacks(); + fileSystem.OnFileExists = () => true; + fileSystem.OnOpenFileStream = (path, fileMode, fileAccess) => + { + if (fileAccess == FileAccess.Write) + { + return new MemoryStream(); + } - [OneTimeTearDown] - public void Teardown() - { - Directory.Delete(this.tempFolder, true); + throw new FileNotFoundException(); + }; + + MockHttpGitObjects httpObjects = new MockHttpGitObjects(); + using (httpObjects.InputStream = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(ValidTestObjectFileContents))) + { + httpObjects.MediaType = GVFSConstants.MediaTypes.LooseObjectMediaType; + GVFSGitObjects dut = this.CreateTestableGVFSGitObjects(httpObjects, fileSystem); + + dut.TryCopyBlobContentStream(ValidTestObjectFileContents, new CancellationToken(), (stream, length) => Assert.Fail("Should not be able to call copy stream callback")) + .ShouldEqual(false); + } } [TestCase] public void SucceedsForNormalLookingLooseObjectDownloads() { + MockFileSystemWithCallbacks fileSystem = new Mock.FileSystem.MockFileSystemWithCallbacks(); + fileSystem.OnFileExists = () => true; + fileSystem.OnOpenFileStream = (path, mode, access) => new MemoryStream(); MockHttpGitObjects httpObjects = new MockHttpGitObjects(); using (httpObjects.InputStream = new MemoryStream(System.Text.Encoding.ASCII.GetBytes(ValidTestObjectFileContents))) { httpObjects.MediaType = GVFSConstants.MediaTypes.LooseObjectMediaType; - GVFSGitObjects dut = this.CreateTestableGVFSGitObjects(httpObjects); + GVFSGitObjects dut = this.CreateTestableGVFSGitObjects(httpObjects, fileSystem); - dut.TryDownloadAndSaveObject(ValidTestObjectFileContents.Substring(0, 2), ValidTestObjectFileContents.Substring(2)) - .ShouldEqual(true); + dut.TryDownloadAndSaveObject(ValidTestObjectFileContents) + .ShouldEqual(GitObjects.DownloadAndSaveObjectResult.Success); } } @@ -55,7 +75,7 @@ public void FailsZeroByteLooseObjectsDownloads() this.AssertRetryableExceptionOnDownload( new MemoryStream(), GVFSConstants.MediaTypes.LooseObjectMediaType, - gitObjects => gitObjects.TryDownloadAndSaveObject("aa", "bbcc")); + gitObjects => gitObjects.TryDownloadAndSaveObject("aabbcc")); } [TestCase] @@ -65,7 +85,7 @@ public void FailsNullByteLooseObjectsDownloads() this.AssertRetryableExceptionOnDownload( new MemoryStream(new byte[256]), GVFSConstants.MediaTypes.LooseObjectMediaType, - gitObjects => gitObjects.TryDownloadAndSaveObject("aa", "bbcc")); + gitObjects => gitObjects.TryDownloadAndSaveObject("aabbcc")); } [TestCase] @@ -75,7 +95,7 @@ public void FailsZeroBytePackDownloads() this.AssertRetryableExceptionOnDownload( new MemoryStream(), GVFSConstants.MediaTypes.PackFileMediaType, - gitObjects => gitObjects.TryDownloadAndSaveCommit("object0", 0)); + gitObjects => gitObjects.TryEnsureCommitIsLocal("object0", 0)); } [TestCase] @@ -85,7 +105,7 @@ public void FailsNullBytePackDownloads() this.AssertRetryableExceptionOnDownload( new MemoryStream(new byte[256]), GVFSConstants.MediaTypes.PackFileMediaType, - gitObjects => gitObjects.TryDownloadAndSaveCommit("object0", 0)); + gitObjects => gitObjects.TryEnsureCommitIsLocal("object0", 0)); } private void AssertRetryableExceptionOnDownload( @@ -96,18 +116,27 @@ public void FailsNullBytePackDownloads() MockHttpGitObjects httpObjects = new MockHttpGitObjects(); httpObjects.InputStream = inputStream; httpObjects.MediaType = mediaType; - GVFSGitObjects gitObjects = this.CreateTestableGVFSGitObjects(httpObjects); + MockFileSystemWithCallbacks fileSystem = new MockFileSystemWithCallbacks(); - Assert.Throws(() => download(gitObjects)); - inputStream.Dispose(); + using (ReusableMemoryStream downloadDestination = new ReusableMemoryStream(string.Empty)) + { + fileSystem.OnFileExists = () => false; + fileSystem.OnOpenFileStream = (path, mode, access) => downloadDestination; + + GVFSGitObjects gitObjects = this.CreateTestableGVFSGitObjects(httpObjects, fileSystem); + + Assert.Throws(() => download(gitObjects)); + inputStream.Dispose(); + } } - private GVFSGitObjects CreateTestableGVFSGitObjects(MockHttpGitObjects httpObjects) + private GVFSGitObjects CreateTestableGVFSGitObjects(MockHttpGitObjects httpObjects, MockFileSystemWithCallbacks fileSystem) { MockTracer tracer = new MockTracer(); - GVFSEnlistment enlistment = new GVFSEnlistment(this.tempFolder, "https://fakeRepoUrl", "fakeGitBinPath", gvfsHooksRoot: null); + GVFSEnlistment enlistment = new GVFSEnlistment(TestEnlistmentRoot, "https://fakeRepoUrl", "fakeGitBinPath", gvfsHooksRoot: null); + GitRepo repo = new GitRepo(tracer, enlistment, fileSystem, () => new MockLibGit2Repo(tracer)); - GVFSContext context = new GVFSContext(tracer, null, null, enlistment); + GVFSContext context = new GVFSContext(tracer, fileSystem, repo, enlistment); GVFSGitObjects dut = new GVFSGitObjects(context, httpObjects); return dut; } @@ -126,7 +155,7 @@ public MockHttpGitObjects() } private MockHttpGitObjects(MockEnlistment enlistment) - : base(new MockTracer(), enlistment, new MockCacheServerInfo(), new RetryConfig()) + : base(new MockTracer(), enlistment, new MockCacheServerInfo(), new RetryConfig(maxRetries: 1)) { } @@ -148,14 +177,8 @@ public static MemoryStream GetRandomStream(int size) public override RetryWrapper.InvocationResult TryDownloadLooseObject( string objectId, - Func.CallbackResult> onSuccess) - { - return this.TryDownloadObjects(new[] { objectId }, 0, onSuccess, null, false); - } - - public override RetryWrapper.InvocationResult TryDownloadLooseObject( - string objectId, - int maxAttempts, + bool retryOnFailure, + CancellationToken cancellationToken, Func.CallbackResult> onSuccess) { return this.TryDownloadObjects(new[] { objectId }, 0, onSuccess, null, false); @@ -174,7 +197,7 @@ public static MemoryStream GetRandomStream(int size) return new RetryWrapper.InvocationResult(0, true, result); } - public override List QueryForFileSizes(IEnumerable objectIds) + public override List QueryForFileSizes(IEnumerable objectIds, CancellationToken cancellationToken) { throw new NotImplementedException(); } diff --git a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs index 82b51d1632..9b0dd3a883 100644 --- a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs @@ -3,6 +3,7 @@ using GVFS.Common.Tracing; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Mock.Git; using NUnit.Framework; @@ -190,7 +191,7 @@ public void TwoThreadsInterleavingFailuresShouldntStompASuccess() private MockGitProcess GetGitProcess() { - MockGitProcess gitProcess = new MockGitProcess(); + MockGitProcess gitProcess = new MockGitProcess(new ConfigurableFileSystem()); gitProcess.SetExpectedCommandResult("config gvfs.FunctionalTests.UserName", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.GenericFailureCode)); gitProcess.SetExpectedCommandResult("config gvfs.FunctionalTests.Password", () => new GitProcess.Result(string.Empty, string.Empty, GitProcess.Result.GenericFailureCode)); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockEnlistment.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockEnlistment.cs index 5d1d17e9e0..e897c78977 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockEnlistment.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockEnlistment.cs @@ -1,13 +1,33 @@ -using System; -using GVFS.Common; +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.UnitTests.Mock.Git; namespace GVFS.UnitTests.Mock.Common { public class MockEnlistment : Enlistment { + private MockGitProcess gitProcess; + public MockEnlistment() - : base("mock:\\path", "mock:\\path", "mock:\\path\\.git\\objects", "mock:\\repoUrl", "mock:\\git", null) + : base("mock:\\path", "mock:\\path", "mock:\\repoUrl", "mock:\\git", null) + { + this.GitObjectsRoot = "mock:\\path\\.git\\objects"; + this.GitPackRoot = "mock:\\path\\.git\\objects\\pack"; + } + + public MockEnlistment(MockGitProcess gitProcess) + : this() + { + this.gitProcess = gitProcess; + } + + public override string GitObjectsRoot { get; } + + public override string GitPackRoot { get; } + + public override GitProcess CreateGitProcess() { + return this.gitProcess ?? new MockGitProcess(); } } } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPhysicalGitObjects.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPhysicalGitObjects.cs index 1a71c35461..10179d05cf 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPhysicalGitObjects.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPhysicalGitObjects.cs @@ -1,4 +1,5 @@ using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Common.Tracing; @@ -9,8 +10,8 @@ namespace GVFS.UnitTests.Mock.Common { public class MockPhysicalGitObjects : GitObjects { - public MockPhysicalGitObjects(ITracer tracer, Enlistment enlistment, GitObjectsHttpRequestor objectRequestor) - : base(tracer, enlistment, objectRequestor) + public MockPhysicalGitObjects(ITracer tracer, PhysicalFileSystem fileSystem, Enlistment enlistment, GitObjectsHttpRequestor objectRequestor) + : base(tracer, enlistment, objectRequestor, fileSystem) { } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs index dadfc41ebf..a332ea91e0 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs @@ -1,31 +1,67 @@ using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; +using System; +using System.Threading; namespace GVFS.UnitTests.Mock.Common { public class MockTracer : ITracer { - public void Dispose() + private AutoResetEvent waitEvent; + + public MockTracer() + { + this.waitEvent = new AutoResetEvent(false); + } + + public string WaitRelatedEventName { get; set; } + + public void WaitForRelatedEvent() { + this.waitEvent.WaitOne(); } public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata) { + if (eventName == this.WaitRelatedEventName) + { + this.waitEvent.Set(); + } } public void RelatedEvent(EventLevel error, string eventName, EventMetadata metadata, Keywords keyword) { + if (eventName == this.WaitRelatedEventName) + { + this.waitEvent.Set(); + } } public void RelatedInfo(string format, params object[] args) { } + + public void RelatedWarning(EventMetadata metadata, string message) + { + } - public void RelatedError(EventMetadata metadata) + public void RelatedWarning(EventMetadata metadata, string message, Keywords keyword) { } - public void RelatedError(EventMetadata metadata, Keywords keyword) + public void RelatedWarning(string message) + { + } + + public void RelatedWarning(string format, params object[] args) + { + } + + public void RelatedError(EventMetadata metadata, string message) + { + } + + public void RelatedError(EventMetadata metadata, string message, Keywords keyword) { } @@ -55,5 +91,23 @@ public ITracer StartActivity(string activityName, EventLevel level, Keywords sta public void Stop(EventMetadata metadata) { } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected void Dispose(bool disposing) + { + if (disposing) + { + if (this.waitEvent != null) + { + this.waitEvent.Dispose(); + this.waitEvent = null; + } + } + } } } \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/ConfigurableFileSystem.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/ConfigurableFileSystem.cs new file mode 100644 index 0000000000..cfe053dc25 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/FileSystem/ConfigurableFileSystem.cs @@ -0,0 +1,50 @@ +using GVFS.Common.FileSystem; +using GVFS.Tests.Should; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.UnitTests.Mock.FileSystem +{ + public class ConfigurableFileSystem : PhysicalFileSystem + { + public ConfigurableFileSystem() + { + this.ExpectedFiles = new Dictionary(); + this.ExpectedDirectories = new HashSet(); + } + + public Dictionary ExpectedFiles { get; } + public HashSet ExpectedDirectories { get; } + + public override void CreateDirectory(string path) + { + } + + public override void MoveAndOverwriteFile(string sourceFileName, string destinationFilename) + { + ReusableMemoryStream source; + this.ExpectedFiles.TryGetValue(sourceFileName, out source).ShouldEqual(true, "Source file does not exist: " + sourceFileName); + this.ExpectedFiles.ContainsKey(destinationFilename).ShouldEqual(true, "MoveAndOverwriteFile expects the destination file to exist: " + destinationFilename); + + this.ExpectedFiles.Remove(sourceFileName); + this.ExpectedFiles[destinationFilename] = source; + } + + public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options) + { + ReusableMemoryStream stream; + this.ExpectedFiles.TryGetValue(path, out stream).ShouldEqual(true, "Unexpected access of file: " + path); + return stream; + } + + public override bool FileExists(string path) + { + return this.ExpectedFiles.ContainsKey(path); + } + + public override bool DirectoryExists(string path) + { + return this.ExpectedDirectories.Contains(path); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MassiveMockFileSystem.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MassiveMockFileSystem.cs deleted file mode 100644 index 27dc4597a2..0000000000 --- a/GVFS/GVFS.UnitTests/Mock/FileSystem/MassiveMockFileSystem.cs +++ /dev/null @@ -1,129 +0,0 @@ -using GVFS.Common; -using GVFS.Common.FileSystem; -using GVFS.Tests.Should; -using Microsoft.Win32.SafeHandles; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; - -namespace GVFS.UnitTests.Mock.FileSystem -{ - /// - /// Intentionally stateless mockup of a large physical directory structure. - /// - public class MassiveMockFileSystem : PhysicalFileSystem - { - public const int FoldersPerFolder = 10; - private static Random randy = new Random(0); - private string rootPath; - private int maxDepth; - - public MassiveMockFileSystem(string rootPath, int maxDepth) - { - this.rootPath = rootPath; - this.maxDepth = maxDepth; - } - - public int MaxTreeSize - { - get { return Enumerable.Range(0, this.maxDepth + 1).Sum(i => (int)Math.Pow(FoldersPerFolder, i)); } - } - - public static string RandomPath(int maxDepth) - { - string output = string.Empty; - int depth = randy.Next(1, maxDepth + 1); - for (int i = 0; i < depth; ++i) - { - char letter = (char)randy.Next('a', 'a' + FoldersPerFolder); - output = Path.Combine(output, letter.ToString()); - } - - return output; - } - - public override void WriteAllText(string path, string contents) - { - } - - public override string ReadAllText(string path) - { - return string.Empty; - } - - public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode) - { - return this.OpenFileStream(path, fileMode, fileAccess, NativeMethods.FileAttributes.FILE_ATTRIBUTE_NORMAL, shareMode); - } - - public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, NativeMethods.FileAttributes attributes, FileShare shareMode) - { - return new MemoryStream(); - } - - public override bool FileExists(string path) - { - return false; - } - - public override void CreateDirectory(string path) - { - throw new NotImplementedException(); - } - - public override void DeleteDirectory(string path, bool recursive = false) - { - } - - public override SafeFileHandle LockDirectory(string path) - { - return new SafeFileHandle(IntPtr.Zero, false); - } - - public override IEnumerable ItemsInDirectory(string path) - { - path.StartsWith(this.rootPath).ShouldEqual(true); - - if (path.Count(c => c == '\\') <= this.maxDepth) - { - for (char c = 'a'; c < 'a' + FoldersPerFolder; ++c) - { - yield return new DirectoryItemInfo - { - Name = c.ToString(), - FullName = Path.Combine(path, c.ToString()), - IsDirectory = true, - Length = 0 - }; - } - } - } - - public override FileProperties GetFileProperties(string path) - { - return new FileProperties(FileAttributes.Directory, DateTime.Now, DateTime.Now, DateTime.Now, 0); - } - - public override void CopyFile(string sourcePath, string destinationPath, bool overwrite) - { - throw new NotImplementedException(); - } - - public override void DeleteFile(string path) - { - throw new NotImplementedException(); - } - - public override SafeHandle OpenFile(string path, FileMode fileMode, FileAccess fileAccess, FileAttributes attributes, FileShare shareMode) - { - throw new NotImplementedException(); - } - - public override IEnumerable ReadLines(string path) - { - throw new NotImplementedException(); - } - } -} diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystem.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystem.cs index da46342c19..ace52b5d61 100644 --- a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystem.cs +++ b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystem.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices; namespace GVFS.UnitTests.Mock.FileSystem { @@ -35,15 +34,43 @@ public override void DeleteFile(string path) this.RootDirectory.RemoveFile(path); } - - public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode) + + public override void MoveAndOverwriteFile(string sourcePath, string destinationPath) { - return this.OpenFileStream(path, fileMode, fileAccess, NativeMethods.FileAttributes.FILE_ATTRIBUTE_NORMAL, shareMode); - } + if (sourcePath == null || destinationPath == null) + { + throw new ArgumentNullException(); + } - public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, NativeMethods.FileAttributes attributes, FileShare shareMode) + MockFile sourceFile = this.RootDirectory.FindFile(sourcePath); + MockFile destinationFile = this.RootDirectory.FindFile(destinationPath); + if (sourceFile == null) + { + throw new FileNotFoundException(); + } + + if (destinationFile != null) + { + this.RootDirectory.RemoveFile(destinationPath); + } + + this.WriteAllText(destinationPath, this.ReadAllText(sourcePath)); + this.RootDirectory.RemoveFile(sourcePath); + } + + public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options) { MockFile file = this.RootDirectory.FindFile(path); + if (fileMode == FileMode.Create) + { + if (file != null) + { + this.RootDirectory.RemoveFile(path); + } + + return this.CreateAndOpenFileStream(path); + } + if (fileMode == FileMode.OpenOrCreate) { if (file == null) @@ -59,23 +86,6 @@ public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess return file.GetContentStream(); } - public override SafeHandle OpenFile(string path, FileMode fileMode, FileAccess fileAccess, FileAttributes attributes, FileShare shareMode) - { - if (fileMode == FileMode.Create) - { - MockFile newFile = this.RootDirectory.CreateFile(path); - FileProperties newProperties = new FileProperties( - attributes, - newFile.FileProperties.CreationTimeUTC, - newFile.FileProperties.LastAccessTimeUTC, - newFile.FileProperties.LastWriteTimeUTC, - newFile.FileProperties.Length); - newFile.FileProperties = newProperties; - } - - return new MockSafeHandle(path, this.OpenFileStream(path, fileMode, fileAccess, shareMode)); - } - public override void WriteAllText(string path, string contents) { MockFile file = new MockFile(path, contents); @@ -106,7 +116,7 @@ public override IEnumerable ReadLines(string path) public override void CreateDirectory(string path) { - throw new NotImplementedException(); + this.RootDirectory.CreateDirectory(path); } public override void DeleteDirectory(string path, bool recursive = false) diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystemWithCallbacks.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystemWithCallbacks.cs new file mode 100644 index 0000000000..35f190bbe9 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockFileSystemWithCallbacks.cs @@ -0,0 +1,61 @@ +using GVFS.Common.FileSystem; +using NUnit.Framework; +using System; +using System.IO; + +namespace GVFS.UnitTests.Mock.FileSystem +{ + public class MockFileSystemWithCallbacks : PhysicalFileSystem + { + public Func OnFileExists { get; set; } + + public Func OnOpenFileStream { get; set; } + + public override FileProperties GetFileProperties(string path) + { + throw new InvalidOperationException("GetFileProperties has not been implemented."); + } + + public override bool FileExists(string path) + { + if (this.OnFileExists == null) + { + throw new InvalidOperationException("OnFileExists should be set if it is expected to be called."); + } + + return this.OnFileExists(); + } + + public override Stream OpenFileStream(string path, FileMode fileMode, FileAccess fileAccess, FileShare shareMode, FileOptions options) + { + if (this.OnOpenFileStream == null) + { + throw new InvalidOperationException("OnOpenFileStream should be set if it is expected to be called."); + } + + return this.OnOpenFileStream(path, fileMode, fileAccess); + } + + public override void WriteAllText(string path, string contents) + { + } + + public override string ReadAllText(string path) + { + throw new InvalidOperationException("ReadAllText has not been implemented."); + } + + public override void DeleteFile(string path) + { + } + + public override void DeleteDirectory(string path, bool recursive = false) + { + throw new InvalidOperationException("DeleteDirectory has not been implemented."); + } + + public override void CreateDirectory(string path) + { + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockSafeHandle.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockSafeHandle.cs deleted file mode 100644 index e6872858cb..0000000000 --- a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockSafeHandle.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.IO; -using System.Runtime.InteropServices; - -namespace GVFS.UnitTests.Mock.FileSystem -{ - /// - /// A "SafeHandle" object to represent fake file contents during native file system calls - /// - public class MockSafeHandle : SafeHandle - { - public MockSafeHandle(string filePath, Stream fileContents) : base(IntPtr.Zero, false) - { - this.FilePath = filePath; - this.FileContents = fileContents; - } - - public string FilePath { get; } - - public Stream FileContents { get; } - - public override bool IsInvalid - { - get { return false; } - } - - protected override bool ReleaseHandle() - { - this.FileContents.Dispose(); - return true; - } - } -} diff --git a/GVFS/GVFS.UnitTests/Mock/GVFS.GvFlt/DotGit/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/GVFS.GvFlt/DotGit/MockGitIndexProjection.cs new file mode 100644 index 0000000000..c1d98b4630 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/GVFS.GvFlt/DotGit/MockGitIndexProjection.cs @@ -0,0 +1,252 @@ +using GVFS.Common; +using GVFS.GVFlt; +using GVFS.GVFlt.DotGit; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace GVFS.UnitTests.Mock.GVFS.GvFlt.DotGit +{ + public class MockGitIndexProjection : GitIndexProjection + { + private ConcurrentHashSet projectedFiles; + + private ManualResetEvent unblockGetProjectedItems; + private ManualResetEvent waitForGetProjectedItems; + + private ManualResetEvent unblockIsPathProjected; + private ManualResetEvent waitForIsPathProjected; + + private ManualResetEvent unblockGetProjectedFileInfo; + private ManualResetEvent waitForGetProjectedFileInfo; + + private AutoResetEvent placeholderCreated; + + public MockGitIndexProjection(IEnumerable projectedFiles) + { + this.projectedFiles = new ConcurrentHashSet(); + foreach (string entry in projectedFiles) + { + this.projectedFiles.Add(entry); + } + + this.PlaceholdersCreated = new ConcurrentHashSet(); + + this.unblockGetProjectedItems = new ManualResetEvent(true); + this.waitForGetProjectedItems = new ManualResetEvent(true); + + this.unblockIsPathProjected = new ManualResetEvent(true); + this.waitForIsPathProjected = new ManualResetEvent(true); + + this.unblockGetProjectedFileInfo = new ManualResetEvent(true); + this.waitForGetProjectedFileInfo = new ManualResetEvent(true); + + this.placeholderCreated = new AutoResetEvent(false); + } + + public bool EnumerationInMemory { get; set; } + + public ConcurrentHashSet PlaceholdersCreated { get; private set; } + + public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } + + public void BlockGetProjectedItems(bool willWaitForRequest) + { + if (willWaitForRequest) + { + this.waitForGetProjectedItems.Reset(); + } + + this.unblockGetProjectedItems.Reset(); + } + + public void UnblockGetProjectedItems() + { + this.unblockGetProjectedItems.Set(); + } + + public void WaitForGetProjectedItems() + { + this.waitForIsPathProjected.WaitOne(); + } + + public void BlockIsPathProjected(bool willWaitForRequest) + { + if (willWaitForRequest) + { + this.waitForIsPathProjected.Reset(); + } + + this.unblockIsPathProjected.Reset(); + } + + public void UnblockIsPathProjected() + { + this.unblockIsPathProjected.Set(); + } + + public void WaitForIsPathProjected() + { + this.waitForIsPathProjected.WaitOne(); + } + + public void BlockGetProjectedFileInfo(bool willWaitForRequest) + { + if (willWaitForRequest) + { + this.waitForGetProjectedFileInfo.Reset(); + } + + this.unblockGetProjectedFileInfo.Reset(); + } + + public void UnblockGetProjectedFileInfo() + { + this.unblockGetProjectedFileInfo.Set(); + } + + public void WaitForGetProjectedFileInfo() + { + this.waitForGetProjectedFileInfo.WaitOne(); + } + + public void WaitForPlaceholderCreate() + { + this.placeholderCreated.WaitOne(); + } + + public override void Initialize(ReliableBackgroundOperations backgroundQueue) + { + } + + public override void InvalidateProjection() + { + } + + public override bool TryGetProjectedItemsFromMemory(string folderPath, out IEnumerable projectedItems) + { + if (this.EnumerationInMemory) + { + projectedItems = this.projectedFiles.Select(name => new GVFltFileInfo(name, size: 0, isFolder: false)).ToList(); + return true; + } + + projectedItems = null; + return false; + } + + public override IEnumerable GetProjectedItems(string folderPath, CancellationToken cancellationToken) + { + this.waitForGetProjectedItems.Set(); + + if (this.ThrowOperationCanceledExceptionOnProjectionRequest) + { + throw new OperationCanceledException(); + } + + this.unblockGetProjectedItems.WaitOne(); + return this.projectedFiles.Select(name => new GVFltFileInfo(name, size: 0, isFolder: false)).ToList(); + } + + public override bool IsPathProjected(string virtualPath, out string fileName, out bool isFolder) + { + this.waitForIsPathProjected.Set(); + this.unblockIsPathProjected.WaitOne(); + + if (this.projectedFiles.Contains(virtualPath)) + { + isFolder = false; + string parentKey; + this.GetChildNameAndParentKey(virtualPath, out fileName, out parentKey); + return true; + } + + fileName = string.Empty; + isFolder = false; + return false; + } + + public override GVFltFileInfo GetProjectedGVFltFileInfoAndSha(CancellationToken cancellationToken, string virtualPath, out string parentFolderPath, out string sha) + { + this.waitForGetProjectedFileInfo.Set(); + + if (this.ThrowOperationCanceledExceptionOnProjectionRequest) + { + throw new OperationCanceledException(); + } + + this.unblockGetProjectedFileInfo.WaitOne(); + + if (this.projectedFiles.Contains(virtualPath)) + { + string childName; + string parentKey; + this.GetChildNameAndParentKey(virtualPath, out childName, out parentKey); + parentFolderPath = parentKey; + sha = "TestSha+" + virtualPath; + return new GVFltFileInfo(childName, size: 0, isFolder: false); + } + + parentFolderPath = null; + sha = null; + return null; + } + + public override void OnPlaceholderFileCreated(string virtualPath, string sha) + { + this.PlaceholdersCreated.Add(virtualPath); + this.placeholderCreated.Set(); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (this.unblockGetProjectedItems != null) + { + this.unblockGetProjectedItems.Dispose(); + this.unblockGetProjectedItems = null; + } + + if (this.waitForGetProjectedItems != null) + { + this.waitForGetProjectedItems.Dispose(); + this.waitForGetProjectedItems = null; + } + + if (this.unblockIsPathProjected != null) + { + this.unblockIsPathProjected.Dispose(); + this.unblockIsPathProjected = null; + } + + if (this.waitForIsPathProjected != null) + { + this.waitForIsPathProjected.Dispose(); + this.waitForIsPathProjected = null; + } + + if (this.unblockGetProjectedFileInfo != null) + { + this.unblockGetProjectedFileInfo.Dispose(); + this.unblockGetProjectedFileInfo = null; + } + + if (this.waitForGetProjectedFileInfo != null) + { + this.waitForGetProjectedFileInfo.Dispose(); + this.waitForGetProjectedFileInfo = null; + } + + if (this.placeholderCreated != null) + { + this.placeholderCreated.Dispose(); + this.placeholderCreated = null; + } + } + + base.Dispose(disposing); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/GVFS.GvFlt/MockReliableBackgroundOperations.cs b/GVFS/GVFS.UnitTests/Mock/GVFS.GvFlt/MockReliableBackgroundOperations.cs new file mode 100644 index 0000000000..e9234ed4a6 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/GVFS.GvFlt/MockReliableBackgroundOperations.cs @@ -0,0 +1,26 @@ +using GVFS.GVFlt; +namespace GVFS.UnitTests.Mock.GVFS.GvFlt +{ + public class MockReliableBackgroundOperations : ReliableBackgroundOperations + { + public MockReliableBackgroundOperations() + { + } + + public override int Count + { + get + { + return 0; + } + } + + public override void Start() + { + } + + public override void Enqueue(GVFltCallbacks.BackgroundGitUpdate backgroundOperation) + { + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockBatchHttpGitObjects.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockBatchHttpGitObjects.cs index b084a1a1de..a971559f81 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockBatchHttpGitObjects.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockBatchHttpGitObjects.cs @@ -8,6 +8,7 @@ using System.IO; using System.Net; using System.Text; +using System.Threading; namespace GVFS.UnitTests.Mock.Git { @@ -21,7 +22,7 @@ public MockBatchHttpGitObjects(ITracer tracer, Enlistment enlistment, Func QueryForFileSizes(IEnumerable objectIds) + public override List QueryForFileSizes(IEnumerable objectIds, CancellationToken cancellationToken) { throw new NotImplementedException(); } @@ -51,13 +52,6 @@ public override GitRefs QueryInfoRefs(string branch) return this.StreamObjects(objectIds, onSuccess, onFailure); } - public override RetryWrapper.InvocationResult TryDownloadLooseObject( - string objectId, - Func.CallbackResult> onSuccess) - { - throw new NotImplementedException(); - } - private RetryWrapper.InvocationResult StreamObjects( IEnumerable objectIds, Func.CallbackResult> onSuccess, diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs index af7b07ba08..094b684f2f 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGVFSGitObjects.cs @@ -1,7 +1,9 @@ using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.Http; +using System; using System.IO; +using System.Threading; namespace GVFS.UnitTests.Mock.Git { @@ -15,7 +17,10 @@ public MockGVFSGitObjects(GVFSContext context, GitObjectsHttpRequestor httpGitOb this.context = context; } - public override bool TryDownloadAndSaveCommit(string objectSha, int commitDepth) + public bool CancelTryCopyBlobContentStream { get; set; } + public uint FileLength { get; set; } + + public override bool TryEnsureCommitIsLocal(string objectSha, int commitDepth) { RetryWrapper.InvocationResult result = this.GitObjectRequestor.TryDownloadObjects( new[] { objectSha }, @@ -35,5 +40,22 @@ public override bool TryDownloadAndSaveCommit(string objectSha, int commitDepth) return result.Succeeded && result.Result.Success; } + + public override bool TryCopyBlobContentStream( + string sha, + CancellationToken cancellationToken, + Action writeAction) + { + if (this.CancelTryCopyBlobContentStream) + { + throw new OperationCanceledException(); + } + + writeAction( + new MemoryStream(new byte[this.FileLength]), + this.FileLength); + + return true; + } } } diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs index ab2e8f78a1..e62afcc709 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs @@ -1,9 +1,11 @@ -using GVFS.Common.Git; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Mock.FileSystem; using System; using System.Collections.Generic; using System.IO; -using GVFS.Tests.Should; namespace GVFS.UnitTests.Mock.Git { @@ -11,10 +13,9 @@ public class MockGitProcess : GitProcess { private Dictionary> expectedCommands = new Dictionary>(); - public MockGitProcess() : base(new MockEnlistment()) + public MockGitProcess(PhysicalFileSystem fileSystem = null) + : base(new MockEnlistment(), fileSystem ?? new ConfigurableFileSystem()) { - // Simulate empty config for cache server tests - this.expectedCommands.Add("config gvfs.cache-server", () => new Result(string.Empty, string.Empty, Result.GenericFailureCode)); } public bool ShouldFail { get; set; } diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitRepo.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitRepo.cs index a74714f1ca..a668dc667b 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockGitRepo.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitRepo.cs @@ -14,7 +14,7 @@ public class MockGitRepo : GitRepo private string rootSha; public MockGitRepo(ITracer tracer, Enlistment enlistment, PhysicalFileSystem fileSystem) - : base() + : base(tracer) { this.rootSha = Guid.NewGuid().ToString(); this.AddTree(this.rootSha, "."); diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockHttpGitObjects.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockHttpGitObjects.cs index ff2be7aecc..8af16b2faf 100644 --- a/GVFS/GVFS.UnitTests/Mock/Git/MockHttpGitObjects.cs +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockHttpGitObjects.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading; namespace GVFS.UnitTests.Mock.Git { @@ -38,7 +39,7 @@ public void AddShaLengths(IEnumerable> shaLengthPairs } } - public override List QueryForFileSizes(IEnumerable objectIds) + public override List QueryForFileSizes(IEnumerable objectIds, CancellationToken cancellationToken) { return objectIds.Select(oid => new GitObjectSize(oid, this.QueryForFileSize(oid))).ToList(); } @@ -72,13 +73,6 @@ public override GitRefs QueryInfoRefs(string branch) return this.GetSingleObject(objectId, onSuccess, onFailure); } - public override RetryWrapper.InvocationResult TryDownloadLooseObject( - string objectId, - Func.CallbackResult> onSuccess) - { - return this.GetSingleObject(objectId, onSuccess, null); - } - private RetryWrapper.InvocationResult GetSingleObject( string objectId, Func.CallbackResult> onSuccess, diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockLibGit2Repo.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockLibGit2Repo.cs new file mode 100644 index 0000000000..effcc4c206 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockLibGit2Repo.cs @@ -0,0 +1,35 @@ +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.IO; + +namespace GVFS.UnitTests.Mock.Git +{ + public class MockLibGit2Repo : LibGit2Repo + { + public MockLibGit2Repo(ITracer tracer) + : base() + { + } + + public override bool CommitAndRootTreeExists(string commitish) + { + return false; + } + + public override bool ObjectExists(string sha) + { + return false; + } + + public override bool TryCopyBlob(string sha, Action writeAction) + { + throw new NotSupportedException(); + } + + public override bool TryGetObjectSize(string sha, out long size) + { + throw new NotSupportedException(); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/GvFlt/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/GvFlt/MockVirtualizationInstance.cs new file mode 100644 index 0000000000..a44120cf1b --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/GvFlt/MockVirtualizationInstance.cs @@ -0,0 +1,196 @@ +using GVFS.Common; +using GvLib; +using System; +using System.Threading; + +namespace GVFS.UnitTests.Mock.GvFlt +{ + public class MockVirtualizationInstance : IVirtualizationInstance, IDisposable + { + private AutoResetEvent commandCompleted; + private AutoResetEvent placeholderCreated; + private ManualResetEvent unblockCreateWriteBuffer; + private ManualResetEvent waitForCreateWriteBuffer; + + public MockVirtualizationInstance() + { + this.commandCompleted = new AutoResetEvent(false); + this.placeholderCreated = new AutoResetEvent(false); + this.CreatedPlaceholders = new ConcurrentHashSet(); + + this.unblockCreateWriteBuffer = new ManualResetEvent(true); + this.waitForCreateWriteBuffer = new ManualResetEvent(true); + + this.WriteFileReturnStatus = NtStatus.Success; + } + + public NtStatus CompletionStatus { get; set; } + + public ConcurrentHashSet CreatedPlaceholders { get; private set; } + + public CancelCommandEvent OnCancelCommand { get; set; } + public EndDirectoryEnumerationEvent OnEndDirectoryEnumeration { get; set; } + public GetDirectoryEnumerationEvent OnGetDirectoryEnumeration { get; set; } + public GetFileStreamEvent OnGetFileStream { get; set; } + public GetPlaceholderInformationEvent OnGetPlaceholderInformation { get; set; } + public NotifyFileHandleClosedEvent OnNotifyFileHandleClosed { get; set; } + public NotifyFileHandleCreatedEvent OnNotifyFileHandleCreated { get; set; } + public NotifyFileRenamedEvent OnNotifyFileRenamed { get; set; } + public NotifyFirstWriteEvent OnNotifyFirstWrite { get; set; } + public NotifyHardlinkCreatedEvent OnNotifyHardlinkCreated { get; set; } + public NotifyPreDeleteEvent OnNotifyPreDelete { get; set; } + public NotifyPreRenameEvent OnNotifyPreRename { get; set; } + public NotifyPreSetHardlinkEvent OnNotifyPreSetHardlink { get; set; } + public QueryFileNameEvent OnQueryFileName { get; set; } + public StartDirectoryEnumerationEvent OnStartDirectoryEnumeration { get; set; } + + public NtStatus WriteFileReturnStatus { get; set; } + + public HResult StartVirtualizationInstance( + string virtualizationRootPath, + uint poolThreadCount, + uint concurrentThreadCount, + bool enableNegativePathCache, + ref uint logicalBytesPerSector, + ref uint writeBufferAlignment) + { + logicalBytesPerSector = 1; + writeBufferAlignment = 1; + + return HResult.Ok; + } + + public HResult StopVirtualizationInstance() + { + throw new NotImplementedException(); + } + + public HResult DetachDriver() + { + throw new NotImplementedException(); + } + + public NtStatus ClearNegativePathCache(ref uint totalEntryNumber) + { + throw new NotImplementedException(); + } + + public NtStatus DeleteFile(string relativePath, UpdateType updateFlags, ref UpdateFailureCause failureReason) + { + throw new NotImplementedException(); + } + + public NtStatus UpdatePlaceholderIfNeeded(string relativePath, DateTime creationTime, DateTime lastAccessTime, DateTime lastWriteTime, DateTime changeTime, uint fileAttributes, long endOfFile, byte[] contentId, byte[] epochId, UpdateType updateFlags, ref UpdateFailureCause failureReason) + { + throw new NotImplementedException(); + } + + public NtStatus CreatePlaceholderAsHardlink(string destinationFileName, string hardLinkTarget) + { + throw new NotImplementedException(); + } + + public WriteBuffer CreateWriteBuffer(uint desiredBufferSize) + { + this.waitForCreateWriteBuffer.Set(); + this.unblockCreateWriteBuffer.WaitOne(); + + return new WriteBuffer(desiredBufferSize, 1); + } + + public NtStatus WriteFile(Guid streamGuid, WriteBuffer buffer, ulong byteOffset, uint length) + { + return this.WriteFileReturnStatus; + } + + public NtStatus WritePlaceholderInformation( + string relativePath, + DateTime creationTime, + DateTime lastAccessTime, + DateTime lastWriteTime, + DateTime changeTime, + uint fileAttributes, + long endOfFile, + bool directory, + byte[] contentId, + byte[] epochId) + { + this.CreatedPlaceholders.Add(relativePath); + this.placeholderCreated.Set(); + return NtStatus.Success; + } + + public void CompleteCommand(int commandId, NtStatus completionStatus) + { + this.CompletionStatus = completionStatus; + this.commandCompleted.Set(); + } + + public NtStatus WaitForCompletionStatus() + { + this.commandCompleted.WaitOne(); + return this.CompletionStatus; + } + + public void WaitForPlaceholderCreate() + { + this.placeholderCreated.WaitOne(); + } + + public void BlockCreateWriteBuffer(bool willWaitForRequest) + { + if (willWaitForRequest) + { + this.waitForCreateWriteBuffer.Reset(); + } + + this.unblockCreateWriteBuffer.Reset(); + } + + public void UnblockCreateWriteBuffer() + { + this.unblockCreateWriteBuffer.Set(); + } + + public void WaitForCreateWriteBuffer() + { + this.waitForCreateWriteBuffer.WaitOne(); + } + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected void Dispose(bool disposing) + { + if (disposing) + { + if (this.commandCompleted != null) + { + this.commandCompleted.Dispose(); + this.commandCompleted = null; + } + + if (this.placeholderCreated != null) + { + this.placeholderCreated.Dispose(); + this.placeholderCreated = null; + } + + if (this.unblockCreateWriteBuffer != null) + { + this.unblockCreateWriteBuffer.Dispose(); + this.unblockCreateWriteBuffer = null; + } + + if (this.waitForCreateWriteBuffer != null) + { + this.waitForCreateWriteBuffer.Dispose(); + this.waitForCreateWriteBuffer = null; + } + } + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/ReusableMemoryStream.cs b/GVFS/GVFS.UnitTests/Mock/ReusableMemoryStream.cs index 099e7ef1b9..10b560b001 100644 --- a/GVFS/GVFS.UnitTests/Mock/ReusableMemoryStream.cs +++ b/GVFS/GVFS.UnitTests/Mock/ReusableMemoryStream.cs @@ -16,6 +16,8 @@ public ReusableMemoryStream(string initialContents) this.length = this.contents.Length; } + public bool TruncateWrites { get; set; } + public override bool CanRead { get { return true; } @@ -46,6 +48,25 @@ public override void Flush() { // noop } + + public string ReadAsString() + { + return Encoding.UTF8.GetString(this.contents, 0, (int)this.length); + } + + public string ReadAt(long position, long length) + { + long lastPosition = this.Position; + + this.Position = position; + + byte[] bytes = new byte[length]; + this.Read(bytes, 0, (int)length); + + this.Position = lastPosition; + + return Encoding.UTF8.GetString(bytes); + } public override int Read(byte[] buffer, int offset, int count) { @@ -103,12 +124,22 @@ public override void Write(byte[] buffer, int offset, int count) this.SetLength(this.position + count); } + if (this.TruncateWrites) + { + count /= 2; + } + Array.Copy(buffer, offset, this.contents, this.position, count); this.position += count; if (this.position > this.length) { this.length = this.position; } + + if (this.TruncateWrites) + { + throw new IOException("Could not complete write"); + } } protected override void Dispose(bool disposing) diff --git a/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs b/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs index 225da7e537..c376435a51 100644 --- a/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs +++ b/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs @@ -4,11 +4,12 @@ using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Mock.Git; using NUnit.Framework; +using System; using System.IO; namespace GVFS.UnitTests.Virtual { - public class CommonRepoSetup + public class CommonRepoSetup : IDisposable { public CommonRepoSetup() { @@ -62,6 +63,30 @@ public CommonRepoSetup() public MockGitRepo Repository { get; private set; } public MockHttpGitObjects HttpObjects { get; private set; } + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected void Dispose(bool disposing) + { + if (disposing) + { + if (this.Context != null) + { + this.Context.Dispose(); + this.Context = null; + } + + if (this.HttpObjects != null) + { + this.HttpObjects.Dispose(); + this.HttpObjects = null; + } + } + } + private static void CreateStandardGitTree(MockGitRepo repository) { string rootSha = repository.GetHeadTreeSha(); @@ -79,7 +104,7 @@ private static void CreateStandardGitTree(MockGitRepo repository) string dupTreeSha = repository.AddChildTree(rootSha, "DupTree"); repository.AddChildBlob(dupTreeSha, "B.1.txt", "B.1 in GitTree"); - + repository.AddChildBlob(rootSha, "C.txt", "C in GitTree"); } } diff --git a/GVFS/GVFS.UnitTests/Virtual/TestsWithCommonRepo.cs b/GVFS/GVFS.UnitTests/Virtual/TestsWithCommonRepo.cs index 7894740a57..7d48092051 100644 --- a/GVFS/GVFS.UnitTests/Virtual/TestsWithCommonRepo.cs +++ b/GVFS/GVFS.UnitTests/Virtual/TestsWithCommonRepo.cs @@ -12,5 +12,14 @@ public virtual void TestSetup() { this.Repo = new CommonRepoSetup(); } + + [TearDown] + public virtual void TestTearDown() + { + if (this.Repo != null) + { + this.Repo.Dispose(); + } + } } } diff --git a/GVFS/GVFS.UnitTests/packages.config b/GVFS/GVFS.UnitTests/packages.config index 00df1c021f..f1ebcd4802 100644 --- a/GVFS/GVFS.UnitTests/packages.config +++ b/GVFS/GVFS.UnitTests/packages.config @@ -1,5 +1,8 @@  + + + diff --git a/GVFS/GVFS/CommandLine/CacheServerVerb.cs b/GVFS/GVFS/CommandLine/CacheServerVerb.cs index 6f7922ce79..2599572c9d 100644 --- a/GVFS/GVFS/CommandLine/CacheServerVerb.cs +++ b/GVFS/GVFS/CommandLine/CacheServerVerb.cs @@ -1,8 +1,9 @@ using CommandLine; using GVFS.Common; -using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Common.Tracing; +using System; +using System.Collections.Generic; using System.Linq; namespace GVFS.CommandLine @@ -14,8 +15,9 @@ public class CacheServerVerb : GVFSVerb.ForExistingEnlistment [Option( "set", + Default = null, Required = false, - HelpText = "Sets the current cache server to the supplied name or url")] + HelpText = "Sets the cache server to the supplied name or url")] public string CacheToSet { get; set; } [Option("get", Required = false, HelpText = "Outputs the current cache server information. This is the default.")] @@ -24,7 +26,7 @@ public class CacheServerVerb : GVFSVerb.ForExistingEnlistment [Option( "list", Required = false, - HelpText = "List available cache servers for the current GVFS enlistment")] + HelpText = "List available cache servers for the remote repo")] public bool ListCacheServers { get; set; } protected override string VerbName @@ -34,51 +36,40 @@ protected override string VerbName protected override void Execute(GVFSEnlistment enlistment) { + this.BlockEmptyCacheServerUrl(this.CacheToSet); + + RetryConfig retryConfig = new RetryConfig(RetryConfig.DefaultMaxRetries, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); + using (ITracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "CacheVerb")) { - RetryConfig retryConfig; - string error; - if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error)) - { - this.ReportErrorAndExit("Failed to determine GVFS timeout and max retries: " + error); - } + GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); - GVFSConfig config; - using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) - { - config = configRequestor.QueryGVFSConfig(); - if (config == null) - { - this.ReportErrorAndExit("Could not query for available cache servers."); - } - } + CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); + string error = null; - CacheServerInfo cache; - if (!string.IsNullOrWhiteSpace(this.CacheToSet)) + if (this.CacheToSet != null) { - if (CacheServerInfo.TryParse(this.CacheToSet, enlistment, config.CacheServers, out cache)) - { - if (!CacheServerInfo.TrySaveToConfig(new GitProcess(enlistment), cache, out error)) - { - this.ReportErrorAndExit("Failed to save cache to config: " + error); - } - } - else + CacheServerInfo cacheServer = cacheServerResolver.ParseUrlOrFriendlyName(this.CacheToSet); + cacheServer = this.ResolveCacheServerUrlIfNeeded(tracer, cacheServer, cacheServerResolver, gvfsConfig); + + if (!cacheServerResolver.TrySaveUrlToLocalConfig(cacheServer, out error)) { - this.ReportErrorAndExit("Unrecognized or invalid cache name or url: " + this.CacheToSet); + this.ReportErrorAndExit("Failed to save cache to config: " + error); } - this.OutputCacheInfo(cache); this.Output.WriteLine("You must remount GVFS for this to take effect."); } else if (this.ListCacheServers) { - if (config.CacheServers.Any()) + List cacheServers = gvfsConfig.CacheServers.ToList(); + + if (cacheServers != null && cacheServers.Any()) { + this.Output.WriteLine(); this.Output.WriteLine("Available cache servers for: " + enlistment.RepoUrl); - foreach (CacheServerInfo cacheServer in config.CacheServers) + foreach (CacheServerInfo cacheServer in cacheServers) { - this.Output.WriteLine("{0, -25} ({1})", cacheServer.Name, cacheServer.Url); + this.Output.WriteLine(cacheServer); } } else @@ -88,19 +79,12 @@ protected override void Execute(GVFSEnlistment enlistment) } else { - if (!CacheServerInfo.TryDetermineCacheServer(null, enlistment, config.CacheServers, out cache, out error)) - { - this.ReportErrorAndExit(error); - } + string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment); + CacheServerInfo cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); - this.OutputCacheInfo(cache); + this.Output.WriteLine("Using cache server: " + cacheServer); } } } - - private void OutputCacheInfo(CacheServerInfo cache) - { - this.Output.WriteLine("Current Cache Server:\t" + cache); - } } } diff --git a/GVFS/GVFS/CommandLine/CloneHelper.cs b/GVFS/GVFS/CommandLine/CloneHelper.cs index 0de487e519..239ea04f12 100644 --- a/GVFS/GVFS/CommandLine/CloneHelper.cs +++ b/GVFS/GVFS/CommandLine/CloneHelper.cs @@ -1,4 +1,5 @@ using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Common.Tracing; @@ -26,8 +27,6 @@ public CloneHelper(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequ public CloneVerb.Result CreateClone(GitRefs refs, string branch) { - GitObjects gitObjects = new GitObjects(this.tracer, this.enlistment, this.objectRequestor); - CloneVerb.Result initRepoResult = this.TryInitRepo(refs, this.enlistment); if (!initRepoResult.Success) { @@ -40,17 +39,27 @@ public CloneVerb.Result CreateClone(GitRefs refs, string branch) return new CloneVerb.Result("Error configuring alternate: " + errorMessage); } - if (!gitObjects.TryDownloadAndSaveCommit(refs.GetTipCommitId(branch), commitDepth: 2)) + PhysicalFileSystem fileSystem = new PhysicalFileSystem(); + GitRepo gitRepo = new GitRepo(this.tracer, this.enlistment, fileSystem); + GVFSGitObjects gitObjects = new GVFSGitObjects(new GVFSContext(this.tracer, fileSystem, gitRepo, this.enlistment), this.objectRequestor); + + if (!gitObjects.TryEnsureCommitIsLocal(refs.GetTipCommitId(branch), commitDepth: 2)) { return new CloneVerb.Result("Could not download tip commits from: " + Uri.EscapeUriString(this.objectRequestor.CacheServer.ObjectsEndpointUrl)); } - GitProcess git = new GitProcess(this.enlistment); - if (!this.SetConfigSettings(git, this.objectRequestor.CacheServer)) + if (!GVFSVerb.TrySetGitConfigSettings(this.enlistment)) { return new CloneVerb.Result("Unable to configure git repo"); } + + CacheServerResolver cacheServerResolver = new CacheServerResolver(this.tracer, this.enlistment); + if (!cacheServerResolver.TrySaveUrlToLocalConfig(this.objectRequestor.CacheServer, out errorMessage)) + { + return new CloneVerb.Result("Unable to configure cache server: " + errorMessage); + } + GitProcess git = new GitProcess(this.enlistment); string originBranchName = "origin/" + branch; GitProcess.Result createBranchResult = git.CreateBranchWithUpstream(branch, originBranchName); if (createBranchResult.HasErrors) @@ -66,7 +75,7 @@ public CloneVerb.Result CreateClone(GitRefs refs, string branch) Path.Combine(this.enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"); - CloneVerb.Result hydrateResult = this.HydrateRootGitAttributes(gitObjects, branch); + CloneVerb.Result hydrateResult = this.HydrateRootGitAttributes(gitObjects, gitRepo, branch); if (!hydrateResult.Success) { return hydrateResult; @@ -110,9 +119,24 @@ public CloneVerb.Result CreateClone(GitRefs refs, string branch) return new CloneVerb.Result(installHooksError); } - using (RepoMetadata repoMetadata = new RepoMetadata(this.enlistment.DotGVFSRoot)) + if (!RepoMetadata.TryInitialize(this.tracer, this.enlistment.DotGVFSRoot, out errorMessage)) { - repoMetadata.SaveCurrentDiskLayoutVersion(); + this.tracer.RelatedError(errorMessage); + return new CloneVerb.Result(errorMessage); + } + + try + { + RepoMetadata.Instance.SaveCurrentDiskLayoutVersion(); + } + catch (Exception e) + { + this.tracer.RelatedError(e.ToString()); + return new CloneVerb.Result(e.Message); + } + finally + { + RepoMetadata.Shutdown(); } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed @@ -137,14 +161,7 @@ private static bool IsForceCheckoutErrorCloneFailure(string checkoutError) return true; } - private bool SetConfigSettings(GitProcess git, CacheServerInfo cacheServer) - { - string error; - return CacheServerInfo.TrySaveToConfig(git, cacheServer, out error) && - GVFSVerb.TrySetGitConfigSettings(git); - } - - private CloneVerb.Result HydrateRootGitAttributes(GitObjects gitObjects, string branch) + private CloneVerb.Result HydrateRootGitAttributes(GVFSGitObjects gitObjects, GitRepo repo, string branch) { List rootEntries = new List(); GitProcess git = new GitProcess(this.enlistment); @@ -164,9 +181,12 @@ private CloneVerb.Result HydrateRootGitAttributes(GitObjects gitObjects, string return new CloneVerb.Result("This branch does not contain a " + GVFSConstants.SpecialGitFiles.GitAttributes + " file in the root folder. This file is required by GVFS clone"); } - if (!gitObjects.TryDownloadAndSaveBlobs(new[] { gitAttributes.TargetSha })) + if (!repo.ObjectExists(gitAttributes.TargetSha)) { - return new CloneVerb.Result("Could not download " + GVFSConstants.SpecialGitFiles.GitAttributes + " file"); + if (gitObjects.TryDownloadAndSaveObject(gitAttributes.TargetSha) != GitObjects.DownloadAndSaveObjectResult.Success) + { + return new CloneVerb.Result("Could not download " + GVFSConstants.SpecialGitFiles.GitAttributes + " file"); + } } return new CloneVerb.Result(true); diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index d273eeb43e..e3313bb2f7 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -34,8 +34,8 @@ public class CloneVerb : GVFSVerb [Option( "cache-server-url", Required = false, - Default = "", - HelpText = "Defines the url of the cache server")] + Default = null, + HelpText = "The url or friendly name of the cache server")] public string CacheServerUrl { get; set; } [Option( @@ -79,12 +79,15 @@ public override void Execute() this.CheckGVFltHealthy(); this.CheckNotInsideExistingRepo(); - + this.BlockEmptyCacheServerUrl(this.CacheServerUrl); + try { GVFSEnlistment enlistment; Result cloneResult = new Result(false); + CacheServerInfo cacheServer = null; + using (JsonEtwTracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "GVFSClone")) { cloneResult = this.TryCreateEnlistment(out enlistment); @@ -94,54 +97,42 @@ public override void Execute() GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Clone), EventLevel.Informational, Keywords.Any); - - string authErrorMessage = null; - if (!this.ShowStatusWhileRunning( - () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), - "Authenticating")) - { - this.ReportErrorAndExit("Unable to clone because authentication failed"); - } - - RetryConfig retryConfig; - string error; - if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error)) - { - this.ReportErrorAndExit("Failed to determine GVFS timeout and max retries: " + error); - } - - retryConfig.Timeout = TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes); - - GVFSConfig gvfsConfig; - CacheServerInfo cacheServer; - using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) - { - gvfsConfig = configRequestor.QueryGVFSConfig(); - } - - if (!CacheServerInfo.TryDetermineCacheServer(this.CacheServerUrl, enlistment, gvfsConfig.CacheServers, out cacheServer, out error)) - { - this.ReportErrorAndExit(error); - } - tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, - cacheServer.Url, + this.CacheServerUrl, + enlistment.GitObjectsRoot, new EventMetadata { { "Branch", this.Branch }, { "SingleBranch", this.SingleBranch }, { "NoMount", this.NoMount }, - { "NoPrefetch", this.NoPrefetch } + { "NoPrefetch", this.NoPrefetch }, + { "Unattended", this.Unattended }, + { "IsElevated", ProcessHelper.IsAdminElevated() }, }); + CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); + cacheServer = cacheServerResolver.ParseUrlOrFriendlyName(this.CacheServerUrl); + this.Output.WriteLine("Clone parameters:"); this.Output.WriteLine(" Repo URL: " + enlistment.RepoUrl); this.Output.WriteLine(" Cache Server: " + cacheServer); this.Output.WriteLine(" Destination: " + enlistment.EnlistmentRoot); - - this.ValidateClientVersions(tracer, enlistment, gvfsConfig); + + string authErrorMessage = null; + if (!this.ShowStatusWhileRunning( + () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), + "Authenticating")) + { + this.ReportErrorAndExit(tracer, "Unable to clone because authentication failed"); + } + + RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); + GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + + cacheServer = this.ResolveCacheServerUrlIfNeeded(tracer, cacheServer, cacheServerResolver, gvfsConfig); + this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: true); this.ShowStatusWhileRunning( () => @@ -162,10 +153,13 @@ public override void Execute() { if (!this.NoPrefetch) { - PrefetchVerb prefetch = new PrefetchVerb(); - prefetch.EnlistmentRootPath = this.EnlistmentRootPath; - prefetch.Commits = true; - prefetch.Execute(); + this.Execute( + verb => + { + verb.Commits = true; + verb.SkipVersionCheck = true; + verb.ResolvedCacheServer = cacheServer; + }); } if (this.NoMount) @@ -175,13 +169,12 @@ public override void Execute() } else { - MountVerb mount = new MountVerb(); - mount.EnlistmentRootPath = this.EnlistmentRootPath; - mount.SkipMountedCheck = true; - mount.SkipVersionCheck = true; - mount.ServiceName = this.ServiceName; - - mount.Execute(); + this.Execute( + verb => + { + verb.SkipMountedCheck = true; + verb.SkipVersionCheck = true; + }); } } else @@ -230,7 +223,7 @@ private Result TryCreateEnlistment(out GVFSEnlistment enlistment) return new Result(GVFSConstants.GitIsNotInstalledError); } - string hooksPath = this.GetGVFSHooksPathAndCheckVersion(); + string hooksPath = this.GetGVFSHooksPathAndCheckVersion(tracer: null); enlistment = new GVFSEnlistment( this.EnlistmentRootPath, diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index c6adaa0ebe..13ef54fadd 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -1,4 +1,5 @@ using CommandLine; +using GVFS.CommandLine.DiskLayoutUpgrades; using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.Http; @@ -48,7 +49,8 @@ protected override void Execute(GVFSEnlistment enlistment) tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, - CacheServerInfo.GetCacheServerValueFromConfig(enlistment), + CacheServerResolver.GetUrlFromConfig(enlistment), + enlistment.GitObjectsRoot, new EventMetadata { { "Confirmed", this.Confirmed }, @@ -86,12 +88,11 @@ protected override void Execute(GVFSEnlistment enlistment) this.Output.WriteLine(); this.Unmount(tracer); - - bool allowUpgrade = false; + string error; - if (!RepoMetadata.CheckDiskLayoutVersion(enlistment.DotGVFSRoot, allowUpgrade, out error)) + if (!DiskLayoutUpgrade.TryCheckDiskLayoutVersion(tracer, enlistment.EnlistmentRoot, out error)) { - this.WriteErrorAndExit(tracer, "GVFS disk layout version doesn't match current version. Run 'gvfs mount' first, then try dehydrate again."); + this.WriteErrorAndExit(tracer, error); } if (this.TryBackupFiles(tracer, enlistment, backupRoot) && @@ -219,6 +220,7 @@ private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string ba string backupGit = Path.Combine(backupRoot, ".git"); string backupInfo = Path.Combine(backupGit, GVFSConstants.DotGit.Info.Name); string backupGvfs = Path.Combine(backupRoot, ".gvfs"); + string backupDatabases = Path.Combine(backupGvfs, GVFSConstants.DotGVFS.Databases.Name); string errorMessage = string.Empty; if (!this.ShowStatusWhileRunning( @@ -228,7 +230,8 @@ private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string ba if (!this.TryIO(tracer, () => Directory.CreateDirectory(backupRoot), "Create backup directory", out ioError) || !this.TryIO(tracer, () => Directory.CreateDirectory(backupGit), "Create backup .git directory", out ioError) || !this.TryIO(tracer, () => Directory.CreateDirectory(backupInfo), "Create backup .git\\info directory", out ioError) || - !this.TryIO(tracer, () => Directory.CreateDirectory(backupGvfs), "Create backup .gvfs directory", out ioError)) + !this.TryIO(tracer, () => Directory.CreateDirectory(backupGvfs), "Create backup .gvfs directory", out ioError) || + !this.TryIO(tracer, () => Directory.CreateDirectory(backupDatabases), "Create backup .gvfs databases directory", out ioError)) { errorMessage = "Failed to create backup folders at " + backupRoot + ": " + ioError; return false; @@ -289,12 +292,16 @@ private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string ba // ... backup the .gvfs hydration-related data structures... if (!this.TryIO( tracer, - () => Directory.Move(Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.BackgroundGitUpdates), Path.Combine(backupGvfs, GVFSConstants.DatabaseNames.BackgroundGitUpdates)), + () => File.Move( + Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundGitOperations), + Path.Combine(backupGvfs, GVFSConstants.DotGVFS.Databases.BackgroundGitOperations)), "Backup the BackgroundGitUpdates database", out errorMessage) || !this.TryIO( tracer, - () => Directory.Move(Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.PlaceholderList), Path.Combine(backupGvfs, GVFSConstants.DatabaseNames.PlaceholderList)), + () => File.Move( + Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList), + Path.Combine(backupGvfs, GVFSConstants.DotGVFS.Databases.PlaceholderList)), "Backup the PlaceholderList database", out errorMessage)) { @@ -374,14 +381,13 @@ private void WriteMessage(ITracer tracer, string message) "Dehydrate", new EventMetadata { - { "Message", message } + { TracingConstants.MessageKey.InfoMessage, message } }); } private void WriteErrorAndExit(ITracer tracer, string message) { - tracer.RelatedError(message); - this.ReportErrorAndExit("ERROR: " + message); + this.ReportErrorAndExit(tracer, "ERROR: " + message); } private ReturnCode ExecuteGVFSVerb(ITracer tracer) @@ -393,13 +399,7 @@ private ReturnCode ExecuteGVFSVerb(ITracer tracer) StringBuilder commandOutput = new StringBuilder(); using (StringWriter writer = new StringWriter(commandOutput)) { - returnCode = GVFSVerb.Execute( - this.EnlistmentRootPath, - verb => - { - verb.Output = writer; - verb.ServiceName = this.ServiceName; - }); + returnCode = this.Execute(verb => verb.Output = writer); } tracer.RelatedEvent( @@ -420,7 +420,8 @@ private ReturnCode ExecuteGVFSVerb(ITracer tracer) { { "Verb", typeof(TVerb).Name }, { "Exception", e.ToString() } - }); + }, + "ExecuteGVFSVerb: Caught exception"); return ReturnCode.GenericError; } @@ -449,8 +450,9 @@ private bool TryIO(ITracer tracer, Action action, string description, out string new EventMetadata { { "Description", description }, - { "Error", error }, - }); + { "Error", error } + }, + "TryIO: Caught exception performing action"); } return false; diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index 50a6216f02..22a0450fcc 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -3,7 +3,6 @@ using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Http; -using GVFS.GVFlt; using Microsoft.Isam.Esent.Collections.Generic; using System; using System.IO; @@ -25,13 +24,6 @@ public class DiagnoseVerb : GVFSVerb.ForExistingEnlistment private TextWriter diagnosticLogFileWriter; - [Option( - GVFSConstants.VerbParameters.Unmount.SkipLock, - Default = false, - Required = false, - HelpText = "Force unmount even if the lock is not available.")] - public bool SkipLock { get; set; } - protected override string VerbName { get { return DiagnoseVerbName; } @@ -61,104 +53,79 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); this.WriteMessage("Repo URL: " + enlistment.RepoUrl); - - string error; - CacheServerInfo cacheServer; - if (CacheServerInfo.TryDetermineCacheServer(null, enlistment, null, out cacheServer, out error)) - { - this.WriteMessage("Cache Server: " + cacheServer); - } - else - { - this.WriteMessage(error); - } + this.WriteMessage("Cache Server: " + CacheServerResolver.GetUrlFromConfig(enlistment)); this.WriteMessage(string.Empty); - this.WriteMessage("Copying .gvfs folder..."); - this.CopyAllFiles(enlistment.EnlistmentRoot, archiveFolderPath, GVFSConstants.DotGVFS.Root, copySubFolders: false); - - this.WriteMessage("Copying GVFlt logs..."); - this.FlushGvFltLogBuffers(); - string system32LogFilesPath = Environment.ExpandEnvironmentVariables(System32LogFilesRoot); - this.CopyAllFiles(system32LogFilesPath, archiveFolderPath, GVFltLogFolderName, copySubFolders: false); - this.LogGvFltTimeout(); - - this.WriteMessage("Checking on GVFS..."); - this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_log.txt"); - ReturnCode statusResult = this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_status.txt"); - - if (statusResult == ReturnCode.Success) - { - this.WriteMessage("GVFS is mounted. Unmounting so we can read files that GVFS has locked..."); - this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_unmount.txt", verb => verb.SkipLock = this.SkipLock); - } - else - { - this.WriteMessage("GVFS was not mounted."); - } - - this.WriteMessage("Checking Defender exclusion..."); this.WriteAntiVirusExclusions(enlistment.EnlistmentRoot, archiveFolderPath, "DefenderExclusionInfo.txt"); - this.WriteMessage("Copying .git folder..."); - this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Root, copySubFolders: false); - this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Hooks.Root, copySubFolders: false); - this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Info.Root, copySubFolders: false); - this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Logs.Root, copySubFolders: true); - this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Refs.Root, copySubFolders: true); - this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Objects.Info.Root, copySubFolders: false); - - this.CopyEsentDatabase( - enlistment.DotGVFSRoot, - Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), - GVFSConstants.DatabaseNames.BackgroundGitUpdates); - this.CopyEsentDatabase( - enlistment.DotGVFSRoot, - Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), - GVFSConstants.DatabaseNames.PlaceholderList); - this.CopyEsentDatabase( - enlistment.DotGVFSRoot, - Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), - GVFSConstants.DatabaseNames.BlobSizes); - this.CopyEsentDatabase( - enlistment.DotGVFSRoot, - Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), - GVFSConstants.DatabaseNames.RepoMetadata); - - this.CopyAllFiles(enlistment.DotGVFSRoot, Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), GVFSConstants.DotGVFS.CorruptObjectsName, copySubFolders: false); - - this.WriteMessage("Copying GVFS.Service logs and data..."); - this.CopyAllFiles( - Paths.GetServiceDataRoot(string.Empty), - archiveFolderPath, - this.ServiceName, - copySubFolders: true); + this.ShowStatusWhileRunning( + () => + this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_status.txt") != ReturnCode.Success || + this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_unmount.txt", verb => verb.SkipLock = true) == ReturnCode.Success, + "Unmounting", + suppressGvfsLogMessage: true); - this.WriteMessage(string.Empty); - this.WriteMessage("Remounting GVFS..."); - ReturnCode mountResult = this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_mount.txt"); - if (mountResult == ReturnCode.Success) - { - this.WriteMessage("Mount succeeded"); - } - else - { - this.WriteMessage("Failed to remount. The reason for failure was captured."); - } + this.ShowStatusWhileRunning( + () => + { + // .gvfs + this.CopyAllFiles(enlistment.EnlistmentRoot, archiveFolderPath, GVFSConstants.DotGVFS.Root, copySubFolders: false); + + // gvflt + this.FlushGvFltLogBuffers(); + string system32LogFilesPath = Environment.ExpandEnvironmentVariables(System32LogFilesRoot); + this.CopyAllFiles(system32LogFilesPath, archiveFolderPath, GVFltLogFolderName, copySubFolders: false); + + // .git + this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Root, copySubFolders: false); + this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Hooks.Root, copySubFolders: false); + this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Info.Root, copySubFolders: false); + this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Logs.Root, copySubFolders: true); + this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Refs.Root, copySubFolders: true); + this.CopyAllFiles(enlistment.WorkingDirectoryRoot, archiveFolderPath, GVFSConstants.DotGit.Objects.Info.Root, copySubFolders: false); + + // databases + this.CopyEsentDatabase(enlistment.DotGVFSRoot, Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), GVFSConstants.DotGVFS.BlobSizesName); + this.CopyAllFiles(enlistment.DotGVFSRoot, Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), GVFSConstants.DotGVFS.Databases.Name, copySubFolders: false); + + // corrupt objects + this.CopyAllFiles(enlistment.DotGVFSRoot, Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), GVFSConstants.DotGVFS.CorruptObjectsName, copySubFolders: false); + + // service + this.CopyAllFiles( + Paths.GetServiceDataRoot(string.Empty), + archiveFolderPath, + this.ServiceName, + copySubFolders: true); + + return true; + }, + "Copying logs"); + + this.ShowStatusWhileRunning( + () => this.RunAndRecordGVFSVerb(archiveFolderPath, "gvfs_mount.txt") == ReturnCode.Success, + "Mounting", + suppressGvfsLogMessage: true); this.CopyAllFiles(enlistment.DotGVFSRoot, Path.Combine(archiveFolderPath, GVFSConstants.DotGVFS.Root), "logs", copySubFolders: false); } string zipFilePath = archiveFolderPath + ".zip"; - ZipFile.CreateFromDirectory(archiveFolderPath, zipFilePath); - PhysicalFileSystem.RecursiveDelete(archiveFolderPath); + this.ShowStatusWhileRunning( + () => + { + ZipFile.CreateFromDirectory(archiveFolderPath, zipFilePath); + PhysicalFileSystem.RecursiveDelete(archiveFolderPath); + + return true; + }, + "Creating zip file", + suppressGvfsLogMessage: true); this.Output.WriteLine(); this.Output.WriteLine("Diagnostics complete. All of the gathered info, as well as all of the output above, is captured in"); this.Output.WriteLine(zipFilePath); - this.Output.WriteLine(); - this.Output.WriteLine("If you are experiencing an issue, please email the GVFS team with your repro steps and include this zip file."); } private void WriteMessage(string message) @@ -178,7 +145,6 @@ private void CopyAllFiles(string sourceRoot, string targetRoot, string folderNam { if (!Directory.Exists(sourceFolder)) { - this.WriteMessage(string.Format("Skipping {0}, folder does not exist", sourceFolder)); return; } @@ -247,7 +213,7 @@ private void RecursiveFileCopyImpl(string sourcePath, string targetPath, bool co } } - private ReturnCode RunAndRecordGVFSVerb(string archiveFolderPath, string outputFileName, Action customConfigureVerb = null) + private ReturnCode RunAndRecordGVFSVerb(string archiveFolderPath, string outputFileName, Action configureVerb = null) where TVerb : GVFSVerb, new() { try @@ -255,16 +221,16 @@ private ReturnCode RunAndRecordGVFSVerb(string archiveFolderPath, string using (FileStream file = new FileStream(Path.Combine(archiveFolderPath, outputFileName), FileMode.CreateNew)) using (StreamWriter writer = new StreamWriter(file)) { - customConfigureVerb = customConfigureVerb ?? new Action(verb => { }); - Action composedVerbConfiguration; - composedVerbConfiguration = verb => - { - customConfigureVerb(verb); - verb.Output = writer; - verb.ServiceName = this.ServiceName; - }; + return this.Execute( + verb => + { + if (configureVerb != null) + { + configureVerb(verb); + } - return GVFSVerb.Execute(this.EnlistmentRootPath, composedVerbConfiguration); + verb.Output = writer; + }); } } catch (Exception e) @@ -329,11 +295,6 @@ private void WriteAntiVirusExclusions(string enlistmentRoot, string archiveFolde using (PersistentDictionary dictionary = new PersistentDictionary( Path.Combine(sourceFolder, databaseName))) { - this.WriteMessage(string.Format( - "Found {0} entries in {1}", - dictionary.Count, - databaseName)); - foreach (TKey key in dictionary.Keys) { writer.Write(key); @@ -355,20 +316,6 @@ private void WriteAntiVirusExclusions(string enlistmentRoot, string archiveFolde this.CopyAllFiles(sourceFolder, targetFolder, databaseName, copySubFolders: false); } - private void LogGvFltTimeout() - { - int gvfltTimeoutMs; - string error; - if (GvFltFilter.TryGetTimeout(out gvfltTimeoutMs, out error)) - { - this.WriteMessage(string.Format("GvFlt timeout (ms) = {0}", gvfltTimeoutMs)); - } - else - { - this.WriteMessage(string.Format("Failed to GvFlt timeout, error: {0}", error)); - } - } - private void FlushGvFltLogBuffers() { try @@ -381,12 +328,6 @@ private void FlushGvFltLogBuffers() "Failed to flush GvFlt log buffers {0}", result)); } - else - { - this.WriteMessage(string.Format( - "Flushed GvFlt log buffer ({0})", - logfileName)); - } } catch (Exception e) { diff --git a/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout10to11Upgrade.cs b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout10to11Upgrade.cs new file mode 100644 index 0000000000..f4344bc9b9 --- /dev/null +++ b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout10to11Upgrade.cs @@ -0,0 +1,26 @@ +using GVFS.Common.Tracing; + +namespace GVFS.CommandLine.DiskLayoutUpgrades +{ + public class DiskLayout10to11Upgrade : DiskLayoutUpgrade + { + protected override int SourceLayoutVersion + { + get { return 10; } + } + + /// + /// Version 10 to 11 only added a new value to BackgroundGitUpdate.OperationType, + /// so we only need to bump the disk layout version version here. + /// + public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) + { + if (!this.TryIncrementDiskLayoutVersion(tracer, enlistmentRoot, this)) + { + return false; + } + + return true; + } + } +} diff --git a/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout7to8Upgrade.cs b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout7to8Upgrade.cs new file mode 100644 index 0000000000..866f2e01d3 --- /dev/null +++ b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout7to8Upgrade.cs @@ -0,0 +1,41 @@ +using GVFS.Common; +using GVFS.Common.Tracing; +using Microsoft.Isam.Esent; +using Microsoft.Isam.Esent.Collections.Generic; +using System.IO; + +namespace GVFS.CommandLine.DiskLayoutUpgrades +{ + public class DiskLayout7to8Upgrade : DiskLayoutUpgrade + { + protected override int SourceLayoutVersion + { + get { return 7; } + } + + /// + /// Version 7 to 8 only added a new value to BackgroundGitUpdate.OperationType, + /// so we only need to bump the ESENT version here. + /// + public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) + { + string dotGVFSRoot = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + string esentRepoMetadata = Path.Combine(dotGVFSRoot, EsentRepoMetadataName); + try + { + using (PersistentDictionary esentMetadata = new PersistentDictionary(esentRepoMetadata)) + { + esentMetadata[DiskLayoutUpgrade.DiskLayoutEsentVersionKey] = "8"; + } + } + catch (EsentException ex) + { + tracer.RelatedError("RepoMetadata appears to be from an older version of GVFS and corrupted: " + ex.Message); + return false; + } + + // Do not call TryIncrementDiskLayoutVersion. It updates the flat repo metadata which does not exist yet. + return true; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout8to9Upgrade.cs b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout8to9Upgrade.cs new file mode 100644 index 0000000000..3df42fd60e --- /dev/null +++ b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout8to9Upgrade.cs @@ -0,0 +1,88 @@ +using GVFS.Common; +using GVFS.Common.Tracing; +using Microsoft.Isam.Esent; +using Microsoft.Isam.Esent.Collections.Generic; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.CommandLine.DiskLayoutUpgrades +{ + public class DiskLayout8to9Upgrade : DiskLayoutUpgrade + { + protected override int SourceLayoutVersion + { + get { return 8; } + } + + /// + /// Rewrites ESENT RepoMetadata DB to flat JSON file + /// + public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) + { + string dotGVFSRoot = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + if (!this.UpdateRepoMetadata(tracer, dotGVFSRoot)) + { + return false; + } + + if (!this.TryIncrementDiskLayoutVersion(tracer, enlistmentRoot, this)) + { + return false; + } + + return true; + } + + private bool UpdateRepoMetadata(ITracer tracer, string dotGVFSRoot) + { + string esentRepoMetadata = Path.Combine(dotGVFSRoot, EsentRepoMetadataName); + if (Directory.Exists(esentRepoMetadata)) + { + try + { + using (PersistentDictionary oldMetadata = new PersistentDictionary(esentRepoMetadata)) + { + string error; + if (!RepoMetadata.TryInitialize(tracer, dotGVFSRoot, out error)) + { + tracer.RelatedError("Could not initialize RepoMetadata: " + error); + return false; + } + + foreach (KeyValuePair kvp in oldMetadata) + { + tracer.RelatedInfo("Copying ESENT entry: {0} = {1}", kvp.Key, kvp.Value); + RepoMetadata.Instance.SetEntry(kvp.Key, kvp.Value); + } + } + } + catch (IOException ex) + { + tracer.RelatedError("Could not write to new repo metadata: " + ex.Message); + return false; + } + catch (EsentException ex) + { + tracer.RelatedError("RepoMetadata appears to be from an older version of GVFS and corrupted: " + ex.Message); + return false; + } + + string backupName; + if (this.TryRenameFolderForDelete(tracer, esentRepoMetadata, out backupName)) + { + // If this fails, we leave behind cruft, but there's no harm because we renamed. + this.TryDeleteFolder(tracer, backupName); + return true; + } + else + { + // To avoid double upgrading, we should rollback if we can't rename the old data + this.TryDeleteFile(tracer, RepoMetadata.Instance.DataFilePath); + return false; + } + } + + return true; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout9to10Upgrade.cs b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout9to10Upgrade.cs new file mode 100644 index 0000000000..074e9413fd --- /dev/null +++ b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayout9to10Upgrade.cs @@ -0,0 +1,175 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using GVFS.GVFlt; +using Microsoft.Isam.Esent; +using Microsoft.Isam.Esent.Collections.Generic; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.CommandLine.DiskLayoutUpgrades +{ + public class DiskLayout9to10Upgrade : DiskLayoutUpgrade + { + private const string EsentBackgroundOpsFolder = "BackgroundGitUpdates"; + private const string EsentPlaceholderListFolder = "PlaceholderList"; + + protected override int SourceLayoutVersion + { + get { return 9; } + } + + /// + /// Rewrites ESENT BackgroundGitUpdates and PlaceholderList DBs to flat formats + /// + public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) + { + string dotGVFSRoot = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + if (!this.UpdateBackgroundOperations(tracer, dotGVFSRoot)) + { + return false; + } + + if (!this.UpdatePlaceholderList(tracer, dotGVFSRoot)) + { + return false; + } + + if (!this.TryIncrementDiskLayoutVersion(tracer, enlistmentRoot, this)) + { + return false; + } + + return true; + } + + private bool UpdatePlaceholderList(ITracer tracer, string dotGVFSRoot) + { + string esentPlaceholderFolder = Path.Combine(dotGVFSRoot, EsentPlaceholderListFolder); + if (Directory.Exists(esentPlaceholderFolder)) + { + string newPlaceholderFolder = Path.Combine(dotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList); + try + { + using (PersistentDictionary oldPlaceholders = + new PersistentDictionary(esentPlaceholderFolder)) + { + string error; + PlaceholderListDatabase newPlaceholders; + if (!PlaceholderListDatabase.TryCreate( + tracer, + newPlaceholderFolder, + new PhysicalFileSystem(), + out newPlaceholders, + out error)) + { + tracer.RelatedError("Failed to create new placeholder database: " + error); + return false; + } + + using (newPlaceholders) + { + List data = new List(); + foreach (KeyValuePair kvp in oldPlaceholders) + { + tracer.RelatedInfo("Copying ESENT entry: {0} = {1}", kvp.Key, kvp.Value); + data.Add(new PlaceholderListDatabase.PlaceholderData(path: kvp.Key, sha: kvp.Value)); + } + + newPlaceholders.WriteAllEntriesAndFlush(data); + } + } + } + catch (IOException ex) + { + tracer.RelatedError("Could not write to new placeholder database: " + ex.Message); + return false; + } + catch (EsentException ex) + { + tracer.RelatedError("Placeholder database appears to be from an older version of GVFS and corrupted: " + ex.Message); + return false; + } + + string backupName; + if (this.TryRenameFolderForDelete(tracer, esentPlaceholderFolder, out backupName)) + { + // If this fails, we leave behind cruft, but there's no harm because we renamed. + this.TryDeleteFolder(tracer, backupName); + return true; + } + else + { + // To avoid double upgrading, we should rollback if we can't rename the old data + this.TryDeleteFile(tracer, RepoMetadata.Instance.DataFilePath); + return false; + } + } + + return true; + } + + private bool UpdateBackgroundOperations(ITracer tracer, string dotGVFSRoot) + { + string esentBackgroundOpsFolder = Path.Combine(dotGVFSRoot, EsentBackgroundOpsFolder); + if (Directory.Exists(esentBackgroundOpsFolder)) + { + string newBackgroundOpsFolder = Path.Combine(dotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundGitOperations); + try + { + using (PersistentDictionary oldBackgroundOps = + new PersistentDictionary(esentBackgroundOpsFolder)) + { + string error; + BackgroundGitUpdateQueue newBackgroundOps; + if (!BackgroundGitUpdateQueue.TryCreate( + tracer, + newBackgroundOpsFolder, + new PhysicalFileSystem(), + out newBackgroundOps, + out error)) + { + tracer.RelatedError("Failed to create new background operations folder: " + error); + return false; + } + + using (newBackgroundOps) + { + foreach (KeyValuePair kvp in oldBackgroundOps) + { + tracer.RelatedInfo("Copying ESENT entry: {0} = {1}", kvp.Key, kvp.Value); + newBackgroundOps.EnqueueAndFlush(kvp.Value); + } + } + } + } + catch (IOException ex) + { + tracer.RelatedError("Could not write to new background operations: " + ex.Message); + return false; + } + catch (EsentException ex) + { + tracer.RelatedError("BackgroundOperations appears to be from an older version of GVFS and corrupted: " + ex.Message); + return false; + } + + string backupName; + if (this.TryRenameFolderForDelete(tracer, esentBackgroundOpsFolder, out backupName)) + { + // If this fails, we leave behind cruft, but there's no harm because we renamed. + this.TryDeleteFolder(tracer, backupName); + return true; + } + else + { + // To avoid double upgrading, we should rollback if we can't rename the old data + this.TryDeleteFile(tracer, RepoMetadata.Instance.DataFilePath); + return false; + } + } + + return true; + } + } +} diff --git a/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayoutUpgrade.cs b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayoutUpgrade.cs new file mode 100644 index 0000000000..362ab56e8c --- /dev/null +++ b/GVFS/GVFS/CommandLine/DiskLayoutUpgrades/DiskLayoutUpgrade.cs @@ -0,0 +1,285 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using Microsoft.Diagnostics.Tracing; +using Microsoft.Isam.Esent.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace GVFS.CommandLine.DiskLayoutUpgrades +{ + public abstract class DiskLayoutUpgrade + { + protected const string EsentRepoMetadataName = "RepoMetadata"; + protected const string DiskLayoutEsentVersionKey = "DiskLayoutVersion"; + + private static readonly Dictionary AllUpgrades = new List() + { + new DiskLayout7to8Upgrade(), + new DiskLayout8to9Upgrade(), + new DiskLayout9to10Upgrade(), + new DiskLayout10to11Upgrade(), + }.ToDictionary( + upgrader => upgrader.SourceLayoutVersion, + upgrader => upgrader); + + protected abstract int SourceLayoutVersion { get; } + + public static bool TryRunAllUpgrades(string enlistmentRoot) + { + using (JsonEtwTracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "DiskLayoutUpgrade")) + { + try + { + DiskLayoutUpgrade upgrade = null; + while (TryFindUpgrade(tracer, enlistmentRoot, out upgrade)) + { + if (upgrade == null) + { + return true; + } + + if (!upgrade.TryUpgrade(tracer, enlistmentRoot)) + { + return false; + } + + if (!CheckLayoutVersionWasIncremented(tracer, enlistmentRoot, upgrade)) + { + return false; + } + } + + return false; + } + catch (Exception e) + { + StartLogFile(enlistmentRoot, tracer); + tracer.RelatedError(e.ToString()); + return false; + } + finally + { + RepoMetadata.Shutdown(); + } + } + } + + public static bool TryCheckDiskLayoutVersion(ITracer tracer, string enlistmentRoot, out string error) + { + error = string.Empty; + int persistedVersionNumber; + try + { + if (TryGetDiskLayoutVersion(tracer, enlistmentRoot, out persistedVersionNumber, out error)) + { + if (persistedVersionNumber < RepoMetadata.DiskLayoutVersion.MinDiskLayoutVersion) + { + error = string.Format( + "Breaking change to GVFS disk layout has been made since cloning. \r\nEnlistment disk layout version: {0} \r\nGVFS disk layout version: {1} \r\nMinimum supported version: {2}", + persistedVersionNumber, + RepoMetadata.DiskLayoutVersion.CurrentDiskLayoutVersion, + RepoMetadata.DiskLayoutVersion.MinDiskLayoutVersion); + + return false; + } + else if (persistedVersionNumber > RepoMetadata.DiskLayoutVersion.MaxDiskLayoutVersion) + { + error = string.Format( + "Changes to GVFS disk layout do not allow mounting after downgrade. Try mounting again using a more recent version of GVFS. \r\nEnlistment disk layout version: {0} \r\nGVFS disk layout version: {1}", + persistedVersionNumber, + RepoMetadata.DiskLayoutVersion.CurrentDiskLayoutVersion); + + return false; + } + else if (persistedVersionNumber != RepoMetadata.DiskLayoutVersion.CurrentDiskLayoutVersion) + { + error = string.Format( + "GVFS disk layout version doesn't match current version. Try running 'gvfs mount' to upgrade. \r\nEnlistment disk layout version: {0} \r\nGVFS disk layout version: {1}", + persistedVersionNumber, + RepoMetadata.DiskLayoutVersion.CurrentDiskLayoutVersion); + + return false; + } + + return true; + } + } + finally + { + RepoMetadata.Shutdown(); + } + + error = "Failed to read disk layout version. " + ConsoleHelper.GetGVFSLogMessage(enlistmentRoot); + return false; + } + + public abstract bool TryUpgrade(ITracer tracer, string enlistmentRoot); + + protected bool TryDeleteFolder(ITracer tracer, string folderName) + { + try + { + PhysicalFileSystem.RecursiveDelete(folderName); + } + catch (Exception e) + { + tracer.RelatedError("Failed to delete folder {0}: {1}", folderName, e.ToString()); + return true; + } + + return true; + } + + protected bool TryDeleteFile(ITracer tracer, string fileName) + { + try + { + File.Delete(fileName); + } + catch (Exception e) + { + tracer.RelatedError("Failed to delete file {0}: {1}", fileName, e.ToString()); + return true; + } + + return true; + } + + protected bool TryRenameFolderForDelete(ITracer tracer, string folderName, out string backupFolder) + { + backupFolder = folderName + ".deleteme"; + + tracer.RelatedInfo("Moving " + folderName + " to " + backupFolder); + + try + { + Directory.Move(folderName, backupFolder); + } + catch (Exception e) + { + tracer.RelatedError("Failed to move {0} to {1}: {2}", folderName, backupFolder, e.ToString()); + return false; + } + + return true; + } + + protected bool TryIncrementDiskLayoutVersion(ITracer tracer, string enlistmentRoot, DiskLayoutUpgrade upgrade) + { + string newVersion = (upgrade.SourceLayoutVersion + 1).ToString(); + string dotGVFSPath = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + string error; + if (!RepoMetadata.TryInitialize(tracer, dotGVFSPath, out error)) + { + tracer.RelatedError("Could not initialize repo metadata: " + error); + return false; + } + + RepoMetadata.Instance.SetEntry(RepoMetadata.Keys.DiskLayoutVersion, newVersion); + tracer.RelatedInfo("Disk layout version is now: " + newVersion); + return true; + } + + private static bool CheckLayoutVersionWasIncremented(JsonEtwTracer tracer, string enlistmentRoot, DiskLayoutUpgrade upgrade) + { + string error; + int actualVersion; + if (!TryGetDiskLayoutVersion(tracer, enlistmentRoot, out actualVersion, out error)) + { + tracer.RelatedError(error); + return false; + } + + int expectedVersion = upgrade.SourceLayoutVersion + 1; + if (actualVersion != expectedVersion) + { + throw new InvalidDataException(string.Format("Disk layout upgrade did not increment layout version. Expected: {0}, Actual: {1}", expectedVersion, actualVersion)); + } + + return true; + } + + private static bool TryFindUpgrade(JsonEtwTracer tracer, string enlistmentRoot, out DiskLayoutUpgrade upgrade) + { + int version; + string error; + if (!TryGetDiskLayoutVersion(tracer, enlistmentRoot, out version, out error)) + { + StartLogFile(enlistmentRoot, tracer); + tracer.RelatedError(error); + upgrade = null; + return false; + } + + if (AllUpgrades.TryGetValue(version, out upgrade)) + { + StartLogFile(enlistmentRoot, tracer); + tracer.RelatedInfo("Upgrading from disk layout {0} to {1}", version, version + 1); + return true; + } + + return true; + } + + private static bool TryGetDiskLayoutVersion(ITracer tracer, string enlistmentRoot, out int version, out string error) + { + string dotGVFSPath = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + string repoMetadataPath = Path.Combine(dotGVFSPath, EsentRepoMetadataName); + if (Directory.Exists(repoMetadataPath)) + { + try + { + using (PersistentDictionary oldMetadata = new PersistentDictionary(repoMetadataPath)) + { + string versionString = oldMetadata[DiskLayoutEsentVersionKey]; + if (!int.TryParse(versionString, out version)) + { + error = "Could not parse version string as integer: " + versionString; + return false; + } + } + } + catch (Exception e) + { + version = 0; + error = e.ToString(); + return false; + } + } + else + { + if (!RepoMetadata.TryInitialize(tracer, dotGVFSPath, out error)) + { + version = 0; + return false; + } + + if (!RepoMetadata.Instance.TryGetOnDiskLayoutVersion(out version, out error)) + { + return false; + } + } + + error = null; + return true; + } + + private static void StartLogFile(string enlistmentRoot, JsonEtwTracer tracer) + { + if (!tracer.HasLogFileEventListener) + { + tracer.AddLogFileEventListener( + GVFSEnlistment.GetNewGVFSLogFileName( + Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.LogPath), + GVFSConstants.LogFileTypes.Upgrade), + EventLevel.Informational, + Keywords.Any); + + tracer.WriteStartEvent(enlistmentRoot, repoUrl: "N/A", cacheServerUrl: "N/A", gitObjectsRoot: "N/A"); + } + } + } +} diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 3dbb4fd53e..a44f8c4f3b 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -15,12 +15,16 @@ namespace GVFS.CommandLine { public abstract class GVFSVerb { + protected const string StartServiceInstructions = "Run 'sc start GVFS.Service' from an elevated command prompt to ensure it is running."; + public GVFSVerb() { this.Output = Console.Out; this.ReturnCode = ReturnCode.Success; this.ServiceName = GVFSConstants.Service.ServiceName; + this.Unattended = GVFSEnlistment.IsUnattended(tracer: null); + this.InitializeDefaultParameterValues(); } @@ -33,6 +37,8 @@ public GVFSVerb() HelpText = "This parameter is reserved for internal use.")] public string ServiceName { get; set; } + public bool Unattended { get; private set; } + public string ServicePipeName { get @@ -47,10 +53,14 @@ public string ServicePipeName protected abstract string VerbName { get; } - public static bool TrySetGitConfigSettings(GitProcess git) + public static bool TrySetGitConfigSettings(Enlistment enlistment) { + string expectedHooksPath = Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Hooks.Root); + expectedHooksPath = expectedHooksPath.Replace('\\', '/'); + Dictionary expectedConfigSettings = new Dictionary { + { "am.keepcr", "true" }, { "core.autocrlf", "false" }, { "core.fscache", "true" }, { "core.gvfs", "true" }, @@ -63,6 +73,7 @@ public static bool TrySetGitConfigSettings(GitProcess git) { "core.bare", "false" }, { "core.logallrefupdates", "true" }, { GitConfigSetting.VirtualizeObjectsGitConfigName, "true" }, + { "core.hookspath", expectedHooksPath }, { "credential.validate", "false" }, { "diff.autoRefreshIndex", "false" }, { "gc.auto", "0" }, @@ -70,9 +81,10 @@ public static bool TrySetGitConfigSettings(GitProcess git) { "index.version", "4" }, { "merge.stat", "false" }, { "receive.autogc", "false" }, - { "am.keepcr", "true" }, }; + GitProcess git = new GitProcess(enlistment); + Dictionary actualConfigSettings; if (!git.TryGetAllLocalConfig(out actualConfigSettings)) { @@ -96,13 +108,21 @@ public static bool TrySetGitConfigSettings(GitProcess git) return true; } - public static ReturnCode Execute( - string enlistmentRootPath, + public abstract void Execute(); + + public virtual void InitializeDefaultParameterValues() + { + } + + protected ReturnCode Execute( Action configureVerb = null) where TVerb : GVFSVerb, new() { TVerb verb = new TVerb(); - verb.EnlistmentRootPath = enlistmentRootPath; + verb.EnlistmentRootPath = this.EnlistmentRootPath; + verb.ServiceName = this.ServiceName; + verb.Unattended = this.Unattended; + if (configureVerb != null) { configureVerb(verb); @@ -119,12 +139,6 @@ public static bool TrySetGitConfigSettings(GitProcess git) return verb.ReturnCode; } - public abstract void Execute(); - - public virtual void InitializeDefaultParameterValues() - { - } - protected bool ShowStatusWhileRunning( Func action, string message, @@ -134,22 +148,30 @@ public virtual void InitializeDefaultParameterValues() action, message, this.Output, - showSpinner: this.Output == Console.Out && !ConsoleHelper.IsConsoleOutputRedirectedToFile(), + showSpinner: !this.Unattended && this.Output == Console.Out && !ConsoleHelper.IsConsoleOutputRedirectedToFile(), gvfsLogEnlistmentRoot: suppressGvfsLogMessage ? null : Paths.GetGVFSEnlistmentRoot(this.EnlistmentRootPath), initialDelayMs: 0); } - protected void ReportErrorAndExit(ReturnCode exitCode, string error, params object[] args) + protected void ReportErrorAndExit(ITracer tracer, ReturnCode exitCode, string error, params object[] args) { - if (error != null) + if (!string.IsNullOrEmpty(error)) { if (args == null || args.Length == 0) { this.Output.WriteLine(error); + if (tracer != null && exitCode != ReturnCode.Success) + { + tracer.RelatedError(error); + } } else { this.Output.WriteLine(error, args); + if (tracer != null && exitCode != ReturnCode.Success) + { + tracer.RelatedError(error, args); + } } } @@ -159,44 +181,75 @@ protected void ReportErrorAndExit(ReturnCode exitCode, string error, params obje protected void ReportErrorAndExit(string error, params object[] args) { - // TODO 1026787: Record these errors in the event log - this.ReportErrorAndExit(ReturnCode.GenericError, error, args); + this.ReportErrorAndExit(tracer: null, exitCode: ReturnCode.GenericError, error: error, args: args); + } + + protected void ReportErrorAndExit(ITracer tracer, string error, params object[] args) + { + this.ReportErrorAndExit(tracer, ReturnCode.GenericError, error, args); } protected void CheckGVFltHealthy() { string error; - string warning; - if (!GvFltFilter.IsHealthy(out error, out warning, tracer: null)) + if (!GvFltFilter.IsHealthy(out error, tracer: null)) + { + this.ReportErrorAndExit(tracer: null, error: error); + } + } + + protected RetryConfig GetRetryConfig(ITracer tracer, GVFSEnlistment enlistment, TimeSpan? timeoutOverride = null) + { + RetryConfig retryConfig; + string error; + if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error)) { - this.ReportErrorAndExit(error); + this.ReportErrorAndExit(tracer, "Failed to determine GVFS timeout and max retries: " + error); } - if (!string.IsNullOrEmpty(warning)) + if (timeoutOverride.HasValue) { - this.Output.WriteLine(warning); + retryConfig.Timeout = timeoutOverride.Value; } + + return retryConfig; } - protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, GVFSConfig gvfsConfig) + protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig) { - this.CheckGitVersion(enlistment); - this.GetGVFSHooksPathAndCheckVersion(); + GVFSConfig gvfsConfig = null; + if (!this.ShowStatusWhileRunning( + () => + { + using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) + { + return configRequestor.TryQueryGVFSConfig(out gvfsConfig); + } + }, + "Querying remote for config", + suppressGvfsLogMessage: true)) + { + this.ReportErrorAndExit("Unable to query /gvfs/config"); + } + + return gvfsConfig; + } + + protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, GVFSConfig gvfsConfig, bool showWarnings) + { + this.CheckGitVersion(tracer, enlistment); + this.GetGVFSHooksPathAndCheckVersion(tracer); this.CheckVolumeSupportsDeleteNotifications(tracer, enlistment); string errorMessage = null; bool errorIsFatal = false; - - if (!this.ShowStatusWhileRunning( - () => this.TryValidateGVFSVersion(enlistment, tracer, gvfsConfig, out errorMessage, out errorIsFatal), - "Validating client version", - suppressGvfsLogMessage: true)) + if (!this.TryValidateGVFSVersion(enlistment, tracer, gvfsConfig, out errorMessage, out errorIsFatal)) { if (errorIsFatal) { - this.ReportErrorAndExit(errorMessage); + this.ReportErrorAndExit(tracer, errorMessage); } - else + else if (showWarnings) { this.Output.WriteLine(); this.Output.WriteLine(errorMessage); @@ -205,31 +258,73 @@ protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, } } - protected string GetGVFSHooksPathAndCheckVersion() + protected string GetGVFSHooksPathAndCheckVersion(ITracer tracer) { string hooksPath = ProcessHelper.WhereDirectory(GVFSConstants.GVFSHooksExecutableName); if (hooksPath == null) { - this.ReportErrorAndExit("Could not find " + GVFSConstants.GVFSHooksExecutableName); + this.ReportErrorAndExit(tracer, "Could not find " + GVFSConstants.GVFSHooksExecutableName); } FileVersionInfo hooksFileVersionInfo = FileVersionInfo.GetVersionInfo(hooksPath + "\\" + GVFSConstants.GVFSHooksExecutableName); string gvfsVersion = ProcessHelper.GetCurrentProcessVersion(); if (hooksFileVersionInfo.ProductVersion != gvfsVersion) { - this.ReportErrorAndExit("GVFS.Hooks version ({0}) does not match GVFS version ({1}).", hooksFileVersionInfo.ProductVersion, gvfsVersion); + this.ReportErrorAndExit(tracer, "GVFS.Hooks version ({0}) does not match GVFS version ({1}).", hooksFileVersionInfo.ProductVersion, gvfsVersion); } return hooksPath; } + protected void BlockEmptyCacheServerUrl(string userInput) + { + if (userInput == null) + { + return; + } + + if (string.IsNullOrWhiteSpace(userInput)) + { + this.ReportErrorAndExit( +@"You must specify a value for the cache server. +You can specify a URL, a name of a configured cache server, or the special names None or Default."); + } + } + + protected CacheServerInfo ResolveCacheServerUrlIfNeeded( + ITracer tracer, + CacheServerInfo cacheServer, + CacheServerResolver cacheServerResolver, + GVFSConfig gvfsConfig) + { + CacheServerInfo resolvedCacheServer = cacheServer; + + if (cacheServer.Url == null) + { + string cacheServerName = cacheServer.Name; + string error = null; + + if (!cacheServerResolver.TryResolveUrlFromRemote( + cacheServerName, + gvfsConfig, + out resolvedCacheServer, + out error)) + { + this.ReportErrorAndExit(tracer, error); + } + } + + this.Output.WriteLine("Using cache server: " + resolvedCacheServer); + return resolvedCacheServer; + } + private void CheckVolumeSupportsDeleteNotifications(ITracer tracer, Enlistment enlistment) { try { if (!NativeMethods.IsFeatureSupportedByVolume(Directory.GetDirectoryRoot(enlistment.EnlistmentRoot), NativeMethods.FileSystemFlags.FILE_RETURNS_CLEANUP_RESULT_INFO)) { - this.ReportErrorAndExit("Error: File system does not support features required by GVFS. Confirm that Windows version is at or beyond that required by GVFS"); + this.ReportErrorAndExit(tracer, "Error: File system does not support features required by GVFS. Confirm that Windows version is at or beyond that required by GVFS"); } } catch (VerbAbortedException) @@ -243,21 +338,20 @@ private void CheckVolumeSupportsDeleteNotifications(ITracer tracer, Enlistment e if (tracer != null) { EventMetadata metadata = new EventMetadata(); - metadata.Add("ErrorMessage", "Failed to determine if file system supports features required by GVFS"); metadata.Add("Exception", e.ToString()); - tracer.RelatedError(metadata); + tracer.RelatedError(metadata, "Failed to determine if file system supports features required by GVFS"); } - this.ReportErrorAndExit("Error: Failed to determine if file system supports features required by GVFS."); + this.ReportErrorAndExit(tracer, "Error: Failed to determine if file system supports features required by GVFS."); } } - private void CheckGitVersion(Enlistment enlistment) + private void CheckGitVersion(ITracer tracer, Enlistment enlistment) { GitProcess.Result versionResult = GitProcess.Version(enlistment); if (versionResult.HasErrors) { - this.ReportErrorAndExit("Error: Unable to retrieve the git version"); + this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } GitVersion gitVersion; @@ -267,19 +361,20 @@ private void CheckGitVersion(Enlistment enlistment) version = version.Substring(12); } - if (!GitVersion.TryParse(version, out gitVersion)) + if (!GitVersion.TryParseVersion(version, out gitVersion)) { - this.ReportErrorAndExit("Error: Unable to parse the git version. {0}", version); + this.ReportErrorAndExit(tracer, "Error: Unable to parse the git version. {0}", version); } if (gitVersion.Platform != GVFSConstants.MinimumGitVersion.Platform) { - this.ReportErrorAndExit("Error: Invalid version of git {0}. Must use gvfs version.", version); + this.ReportErrorAndExit(tracer, "Error: Invalid version of git {0}. Must use gvfs version.", version); } if (gitVersion.IsLessThan(GVFSConstants.MinimumGitVersion)) { this.ReportErrorAndExit( + tracer, "Error: Installed git version {0} is less than the minimum version of {1}.", gitVersion, GVFSConstants.MinimumGitVersion); @@ -313,8 +408,7 @@ private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, G } EventMetadata metadata = new EventMetadata(); - metadata.Add("ErrorMessage", errorMessage); - tracer.RelatedError(metadata, Keywords.Network); + tracer.RelatedError(metadata, errorMessage, Keywords.Network); return false; } diff --git a/GVFS/GVFS/CommandLine/LogVerb.cs b/GVFS/GVFS/CommandLine/LogVerb.cs index e55a3b8ec7..6e6279a016 100644 --- a/GVFS/GVFS/CommandLine/LogVerb.cs +++ b/GVFS/GVFS/CommandLine/LogVerb.cs @@ -39,9 +39,11 @@ public override void Execute() enlistmentRoot, GVFSConstants.DotGVFS.LogPath); this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.Clone), GVFSConstants.LogFileTypes.Clone); - this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.Dehydrate), GVFSConstants.LogFileTypes.Dehydrate); - this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.Mount), GVFSConstants.LogFileTypes.Mount); + + // By using MountPrefix ("mount") DisplayMostRecent will display either mount_verb, mount_upgrade, or mount_process, whichever is more recent + this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.MountPrefix), GVFSConstants.LogFileTypes.MountPrefix); this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.Prefetch), GVFSConstants.LogFileTypes.Prefetch); + this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.Dehydrate), GVFSConstants.LogFileTypes.Dehydrate); this.DisplayMostRecent(gvfsLogsRoot, GetLogFilePatternForType(GVFSConstants.LogFileTypes.Repair), GVFSConstants.LogFileTypes.Repair); string serviceLogsRoot = Paths.GetServiceLogsPath(this.ServiceName); diff --git a/GVFS/GVFS/CommandLine/MountVerb.cs b/GVFS/GVFS/CommandLine/MountVerb.cs index 9bf6c6848b..d7131037b5 100644 --- a/GVFS/GVFS/CommandLine/MountVerb.cs +++ b/GVFS/GVFS/CommandLine/MountVerb.cs @@ -1,10 +1,12 @@ using CommandLine; +using GVFS.CommandLine.DiskLayoutUpgrades; using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Common.NamedPipes; using GVFS.Common.Tracing; +using GVFS.GVFlt.DotGit; using Microsoft.Diagnostics.Tracing; using System; using System.IO; @@ -71,14 +73,18 @@ protected override void PreCreateEnlistment() { if (pipeClient.Connect(500)) { - this.ReportErrorAndExit(ReturnCode.Success, "This repo is already mounted."); + this.ReportErrorAndExit(tracer: null, exitCode: ReturnCode.Success, error: "This repo is already mounted."); } } } + + if (!DiskLayoutUpgrade.TryRunAllUpgrades(enlistmentRoot)) + { + this.ReportErrorAndExit("Failed to upgrade repo disk layout. " + ConsoleHelper.GetGVFSLogMessage(enlistmentRoot)); + } - bool allowUpgrade = true; string error; - if (!RepoMetadata.CheckDiskLayoutVersion(Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root), allowUpgrade, out error)) + if (!DiskLayoutUpgrade.TryCheckDiskLayoutVersion(tracer: null, enlistmentRoot: enlistmentRoot, error: out error)) { this.ReportErrorAndExit("Error: " + error); } @@ -97,13 +103,40 @@ protected override void Execute(GVFSEnlistment enlistment) this.ReportErrorAndExit("Error configuring alternate: " + errorMessage); } + CacheServerInfo cacheServer = CacheServerResolver.GetCacheServerFromConfig(enlistment); + + string mountExeLocation = null; using (JsonEtwTracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "PreMount")) { tracer.AddLogFileEventListener( - GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Mount), + GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.MountVerb), EventLevel.Verbose, Keywords.Any); - + tracer.WriteStartEvent( + enlistment.EnlistmentRoot, + enlistment.RepoUrl, + cacheServer.Url, + enlistment.GitObjectsRoot, + new EventMetadata + { + { "Unattended", this.Unattended }, + { "IsElevated", ProcessHelper.IsAdminElevated() }, + }); + + // TODO 1050199: Once the service is an optional component, GVFS should only attempt to attach + // GvFlt via the service if the service is present\enabled + if (!GvFltFilter.TryAttach(tracer, enlistment.EnlistmentRoot, out errorMessage)) + { + if (!this.ShowStatusWhileRunning( + () => { return this.AttachGvFltThroughService(enlistment, out errorMessage); }, + "Attaching GvFlt to volume")) + { + this.ReportErrorAndExit(tracer, errorMessage); + } + } + + this.CheckAntiVirusExclusion(tracer, enlistment.EnlistmentRoot); + if (!this.SkipVersionCheck) { string authErrorMessage = null; @@ -114,58 +147,83 @@ protected override void Execute(GVFSEnlistment enlistment) this.Output.WriteLine(" WARNING: " + authErrorMessage); this.Output.WriteLine(" Mount will proceed, but new files cannot be accessed until GVFS can authenticate."); } - } - RetryConfig retryConfig = null; - string error; - if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error)) - { - this.ReportErrorAndExit("Failed to determine GVFS timeout and max retries: " + error); - } + RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment); + GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); - GVFSConfig gvfsConfig; - CacheServerInfo cacheServer; - using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) - { - gvfsConfig = configRequestor.QueryGVFSConfig(); + this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: true); + + CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); + cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, gvfsConfig); + this.Output.WriteLine("Configured cache server: " + cacheServer); } - if (!CacheServerInfo.TryDetermineCacheServer(null, enlistment, gvfsConfig.CacheServers, out cacheServer, out error)) + if (!this.ShowStatusWhileRunning( + () => { return this.PerformPreMountValidation(tracer, enlistment, out mountExeLocation, out errorMessage); }, + "Validating repo")) { - this.ReportErrorAndExit(error); + this.ReportErrorAndExit(tracer, errorMessage); } + } - tracer.WriteStartEvent( - enlistment.EnlistmentRoot, - enlistment.RepoUrl, - cacheServer.Url); + if (!this.ShowStatusWhileRunning( + () => { return this.TryMount(enlistment, mountExeLocation, out errorMessage); }, + "Mounting")) + { + this.ReportErrorAndExit(errorMessage); + } - if (!GvFltFilter.TryAttach(tracer, enlistment.EnlistmentRoot, out errorMessage)) + if (!this.Unattended) + { + if (!this.ShowStatusWhileRunning( + () => { return this.RegisterMount(enlistment, out errorMessage); }, + "Registering for automount")) { - if (!this.ShowStatusWhileRunning( - () => { return this.AttachGvFltThroughService(enlistment, out errorMessage); }, - "Attaching GvFlt to volume")) - { - this.ReportErrorAndExit(errorMessage); - } + this.Output.WriteLine(" WARNING: " + errorMessage); } + } + } + + private bool PerformPreMountValidation(ITracer tracer, GVFSEnlistment enlistment, out string mountExeLocation, out string errorMessage) + { + errorMessage = string.Empty; + mountExeLocation = string.Empty; - this.ValidateClientVersions(tracer, enlistment, gvfsConfig); + // We have to parse these parameters here to make sure they are valid before + // handing them to the background process which cannot tell the user when they are bad + EventLevel verbosity; + Keywords keywords; + this.ParseEnumArgs(out verbosity, out keywords); + + mountExeLocation = Path.Combine(ProcessHelper.GetCurrentProcessLocation(), GVFSConstants.MountExecutableName); + if (!File.Exists(mountExeLocation)) + { + errorMessage = "Could not find GVFS.Mount.exe. You may need to reinstall GVFS."; + return false; } - if (!this.ShowStatusWhileRunning( - () => { return this.TryMount(enlistment, out errorMessage); }, - "Mounting")) + GitProcess git = new GitProcess(enlistment); + if (!git.IsValidRepo()) { - this.ReportErrorAndExit(errorMessage); + errorMessage = "The .git folder is missing or has invalid contents"; + return false; } - if (!this.ShowStatusWhileRunning( - () => { return this.RegisterMount(enlistment, out errorMessage); }, - "Registering for automount")) + try { - this.Output.WriteLine(" WARNING: " + errorMessage); + GitIndexProjection.ReadIndex(Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Index)); + } + catch (Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Exception", e.ToString()); + tracer.RelatedError(metadata, "Index validation failed"); + errorMessage = "Index validation failed, run 'gvfs repair' to repair index."; + + return false; } + + return true; } private bool AttachGvFltThroughService(GVFSEnlistment enlistment, out string errorMessage) @@ -179,7 +237,7 @@ private bool AttachGvFltThroughService(GVFSEnlistment enlistment, out string err { if (!client.Connect()) { - errorMessage = "Unable to mount because GVFS.Service is not responding. Run 'sc start GVFS.Service' from an elevated command prompt to ensure it is running."; + errorMessage = "Unable to mount because GVFS.Service is not responding. " + GVFSVerb.StartServiceInstructions; return false; } @@ -221,29 +279,136 @@ private bool AttachGvFltThroughService(GVFSEnlistment enlistment, out string err } } - private bool TryMount(GVFSEnlistment enlistment, out string errorMessage) + private bool ExcludeFromAntiVirusThroughService(string path, out string errorMessage) { - // We have to parse these parameters here to make sure they are valid before - // handing them to the background process which cannot tell the user when they are bad - EventLevel verbosity; - Keywords keywords; - this.ParseEnumArgs(out verbosity, out keywords); + errorMessage = string.Empty; - string mountExeLocation = Path.Combine(ProcessHelper.GetCurrentProcessLocation(), GVFSConstants.MountExecutableName); - if (!File.Exists(mountExeLocation)) + NamedPipeMessages.ExcludeFromAntiVirusRequest request = new NamedPipeMessages.ExcludeFromAntiVirusRequest(); + request.ExclusionPath = path; + + using (NamedPipeClient client = new NamedPipeClient(this.ServicePipeName)) { - errorMessage = "Could not find GVFS.Mount.exe. You may need to reinstall GVFS."; - return false; + if (!client.Connect()) + { + errorMessage = "Unable to exclude from antivirus because GVFS.Service is not responding. " + GVFSVerb.StartServiceInstructions; + return false; + } + + try + { + client.SendRequest(request.ToMessage()); + NamedPipeMessages.Message response = client.ReadResponse(); + if (response.Header == NamedPipeMessages.ExcludeFromAntiVirusRequest.Response.Header) + { + NamedPipeMessages.ExcludeFromAntiVirusRequest.Response message = NamedPipeMessages.ExcludeFromAntiVirusRequest.Response.FromMessage(response); + + if (!string.IsNullOrEmpty(message.ErrorMessage)) + { + errorMessage = message.ErrorMessage; + return false; + } + + return message.State == NamedPipeMessages.CompletionState.Success; + } + else + { + errorMessage = string.Format("GVFS.Service responded with unexpected message: {0}", response); + return false; + } + } + catch (BrokenPipeException e) + { + errorMessage = "Unable to communicate with GVFS.Service: " + e.ToString(); + return false; + } } + } - GitProcess git = new GitProcess(enlistment); - if (!git.IsValidRepo()) + private void CheckAntiVirusExclusion(ITracer tracer, string path) + { + bool isExcluded; + string getError; + if (AntiVirusExclusions.TryGetIsPathExcluded(path, out isExcluded, out getError)) { - errorMessage = "The physical git repo is missing or invalid"; - return false; + if (!isExcluded) + { + if (ProcessHelper.IsAdminElevated()) + { + string addError; + if (AntiVirusExclusions.AddAntiVirusExclusion(path, out addError)) + { + addError = string.Empty; + if (!AntiVirusExclusions.TryGetIsPathExcluded(path, out isExcluded, out getError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("getError", getError); + metadata.Add("path", path); + tracer.RelatedWarning(metadata, "CheckAntiVirusExclusion: Failed to determine if path excluded after adding it"); + } + } + else + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("addError", addError); + metadata.Add("path", path); + tracer.RelatedWarning(metadata, "CheckAntiVirusExclusion: AddAntiVirusExclusion failed"); + } + } + else + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("path", path); + metadata.Add(TracingConstants.MessageKey.InfoMessage, "CheckAntiVirusExclusion: Skipping call to AddAntiVirusExclusion, GVFS is not running with elevation"); + tracer.RelatedEvent(EventLevel.Informational, "CheckAntiVirusExclusion_SkipLocalAdd", metadata); + } + } + } + else + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("getError", getError); + metadata.Add("path", path); + tracer.RelatedWarning(metadata, "CheckAntiVirusExclusion: Failed to determine if path excluded"); + } + + string errorMessage = null; + if (!isExcluded && !this.Unattended) + { + if (this.ShowStatusWhileRunning( + () => { return this.ExcludeFromAntiVirusThroughService(path, out errorMessage); }, + string.Format("Excluding '{0}' from antivirus", path))) + { + isExcluded = true; + } + else + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("errorMessage", errorMessage); + metadata.Add("path", path); + tracer.RelatedWarning(metadata, "CheckAntiVirusExclusion: Failed to exclude path through service"); + } } - this.SetGitConfigSettings(git); + if (!isExcluded) + { + this.Output.WriteLine(); + this.Output.WriteLine("WARNING: Unable to ensure that '{0}' is excluded from antivirus", path); + if (!string.IsNullOrEmpty(errorMessage)) + { + this.Output.WriteLine(errorMessage); + } + + this.Output.WriteLine(); + } + } + + private bool TryMount(GVFSEnlistment enlistment, string mountExeLocation, out string errorMessage) + { + if (!GVFSVerb.TrySetGitConfigSettings(enlistment)) + { + errorMessage = "Unable to configure git repo"; + return false; + } const string ParamPrefix = "--"; ProcessHelper.StartBackgroundProcess( @@ -258,7 +423,7 @@ private bool TryMount(GVFSEnlistment enlistment, out string errorMessage) this.ShowDebugWindow ? ParamPrefix + GVFSConstants.VerbParameters.Mount.DebugWindow : string.Empty), createWindow: this.ShowDebugWindow); - return GVFSEnlistment.WaitUntilMounted(enlistment.EnlistmentRoot, out errorMessage); + return GVFSEnlistment.WaitUntilMounted(enlistment.EnlistmentRoot, this.Unattended, out errorMessage); } private bool RegisterMount(GVFSEnlistment enlistment, out string errorMessage) @@ -319,14 +484,6 @@ private bool RegisterMount(GVFSEnlistment enlistment, out string errorMessage) } } - private void SetGitConfigSettings(GitProcess git) - { - if (!GVFSVerb.TrySetGitConfigSettings(git)) - { - this.ReportErrorAndExit("Unable to configure git repo"); - } - } - private void ParseEnumArgs(out EventLevel verbosity, out Keywords keywords) { if (!Enum.TryParse(this.KeywordsCsv, out keywords)) diff --git a/GVFS/GVFS/CommandLine/PrefetchHelper.cs b/GVFS/GVFS/CommandLine/PrefetchHelper.cs deleted file mode 100644 index ce10278e40..0000000000 --- a/GVFS/GVFS/CommandLine/PrefetchHelper.cs +++ /dev/null @@ -1,52 +0,0 @@ -using GVFS.Common; -using GVFS.Common.Git; -using GVFS.Common.Http; -using GVFS.Common.Tracing; -using System.IO; - -namespace GVFS.CommandLine -{ - public class PrefetchHelper - { - private readonly GitObjects gitObjects; - - public PrefetchHelper(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor) - { - this.gitObjects = new GitObjects(tracer, enlistment, objectRequestor); - } - - public bool TryPrefetchCommitsAndTrees() - { - string[] packs = this.gitObjects.ReadPackFileNames(GVFSConstants.PrefetchPackPrefix); - long max = -1; - foreach (string pack in packs) - { - long? timestamp = GetTimestamp(pack); - if (timestamp.HasValue && timestamp > max) - { - max = timestamp.Value; - } - } - - return this.gitObjects.TryDownloadPrefetchPacks(max); - } - - private static long? GetTimestamp(string packName) - { - string filename = Path.GetFileName(packName); - if (!filename.StartsWith(GVFSConstants.PrefetchPackPrefix)) - { - return null; - } - - string[] parts = filename.Split('-'); - long parsed; - if (parts.Length > 1 && long.TryParse(parts[1], out parsed)) - { - return parsed; - } - - return null; - } - } -} diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index ea31c462c7..264b9a41ad 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -1,6 +1,7 @@ using CommandLine; using FastFetch; using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Common.Tracing; @@ -19,28 +20,34 @@ public class PrefetchVerb : GVFSVerb.ForExistingEnlistment private static readonly int SearchThreadCount = Environment.ProcessorCount; private static readonly int DownloadThreadCount = Environment.ProcessorCount; private static readonly int IndexThreadCount = Environment.ProcessorCount; + + [Option( + "files", + Required = false, + Default = "", + HelpText = "A semicolon-delimited list of files to fetch. Simple prefix wildcards, e.g. *.txt, are supported.")] + public string Files { get; set; } [Option( - 'f', - Parameters.Folders, + "folders", Required = false, - Default = Parameters.DefaultPathWhitelist, - HelpText = "A semicolon-delimited list of paths to fetch")] - public string PathWhitelist { get; set; } + Default = "", + HelpText = "A semicolon-delimited list of folders to fetch. Wildcards are not supported.")] + public string Folders { get; set; } [Option( - Parameters.FoldersList, + "folders-list", Required = false, - Default = Parameters.DefaultPathWhitelistFile, - HelpText = "A file containing line-delimited list of paths to fetch")] - public string PathWhitelistFile { get; set; } + Default = "", + HelpText = "A file containing line-delimited list of folders to fetch. Wildcards are not supported.")] + public string FoldersListFile { get; set; } [Option( 'c', - Parameters.Commits, + "commits", Required = false, Default = false, - HelpText = "Prefetch the latest set of commit and tree packs")] + HelpText = "Fetch the latest set of commit and tree packs. This option cannot be used with any of the file- or folder-related options.")] public bool Commits { get; set; } [Option( @@ -50,17 +57,14 @@ public class PrefetchVerb : GVFSVerb.ForExistingEnlistment HelpText = "Show all outputs on the console in addition to writing them to a log file")] public bool Verbose { get; set; } + public bool SkipVersionCheck { get; set; } + public CacheServerInfo ResolvedCacheServer { get; set; } + protected override string VerbName { get { return PrefetchVerbName; } } - public override void InitializeDefaultParameterValues() - { - this.PathWhitelist = Parameters.DefaultPathWhitelist; - this.PathWhitelistFile = Parameters.DefaultPathWhitelistFile; - } - protected override void Execute(GVFSEnlistment enlistment) { using (JsonEtwTracer tracer = new JsonEtwTracer(GVFSConstants.GVFSEtwProviderName, "Prefetch")) @@ -69,55 +73,65 @@ protected override void Execute(GVFSEnlistment enlistment) { tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any); } - else - { - tracer.AddPrettyConsoleEventListener(EventLevel.Error, Keywords.Any); - } + + string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment); tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.Prefetch), EventLevel.Informational, Keywords.Any); + tracer.WriteStartEvent( + enlistment.EnlistmentRoot, + enlistment.RepoUrl, + cacheServerUrl, + enlistment.GitObjectsRoot); - RetryConfig retryConfig; - string error; - if (!RetryConfig.TryLoadFromGitConfig(tracer, enlistment, out retryConfig, out error)) - { - tracer.RelatedError("Failed to determine GVFS timeout and max retries: " + error); - Environment.Exit((int)ReturnCode.GenericError); - } + RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); - retryConfig.Timeout = TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes); - - CacheServerInfo cache; - if (!CacheServerInfo.TryDetermineCacheServer(null, tracer, enlistment, retryConfig, out cache, out error)) + CacheServerInfo cacheServer = this.ResolvedCacheServer; + if (!this.SkipVersionCheck) { - tracer.RelatedError(error); - Environment.ExitCode = (int)ReturnCode.GenericError; - return; - } + string authErrorMessage; + if (!this.ShowStatusWhileRunning( + () => enlistment.Authentication.TryRefreshCredentials(tracer, out authErrorMessage), + "Authenticating")) + { + this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed"); + } - tracer.WriteStartEvent( - enlistment.EnlistmentRoot, - enlistment.RepoUrl, - cache.Url); + GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + + CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); + cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); + + this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false); + } try { EventMetadata metadata = new EventMetadata(); metadata.Add("Commits", this.Commits); - metadata.Add("PathWhitelist", this.PathWhitelist); - metadata.Add("PathWhitelistFile", this.PathWhitelistFile); + metadata.Add("Files", this.Files); + metadata.Add("Folders", this.Folders); + metadata.Add("FoldersListFile", this.FoldersListFile); tracer.RelatedEvent(EventLevel.Informational, "PerformPrefetch", metadata); - GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cache, retryConfig); + GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig); + if (this.Commits) { - this.PrefetchCommits(tracer, enlistment, objectRequestor); + if (!string.IsNullOrWhiteSpace(this.Files) || + !string.IsNullOrWhiteSpace(this.Folders) || + !string.IsNullOrWhiteSpace(this.FoldersListFile)) + { + this.ReportErrorAndExit(tracer, "You cannot prefetch commits and blobs at the same time."); + } + + this.PrefetchCommits(tracer, enlistment, objectRequestor, cacheServer); } else { - this.PrefetchBlobs(tracer, enlistment, objectRequestor); + this.PrefetchBlobs(tracer, enlistment, objectRequestor, cacheServer); } } catch (VerbAbortedException) @@ -135,9 +149,9 @@ protected override void Execute(GVFSEnlistment enlistment) new EventMetadata { { "Verb", typeof(PrefetchVerb).Name }, - { "ErrorMessage", $"Unhandled {innerException.GetType().Name}: {innerException.Message}" }, { "Exception", innerException.ToString() } - }); + }, + $"Unhandled {innerException.GetType().Name}: {innerException.Message}"); } Environment.ExitCode = (int)ReturnCode.GenericError; @@ -151,21 +165,15 @@ protected override void Execute(GVFSEnlistment enlistment) new EventMetadata { { "Verb", typeof(PrefetchVerb).Name }, - { "ErrorMessage", $"Unhandled {e.GetType().Name}: {e.Message}" }, { "Exception", e.ToString() } - }); + }, + $"Unhandled {e.GetType().Name}: {e.Message}"); } } } - private void PrefetchCommits(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor) + private void PrefetchCommits(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor, CacheServerInfo cacheServer) { - if (!string.IsNullOrEmpty(this.PathWhitelistFile) || - !string.IsNullOrWhiteSpace(this.PathWhitelist)) - { - this.ReportErrorAndExit("Cannot supply both --commits (-c) and --folders (-f)"); - } - if (this.Verbose) { this.TryPrefetchCommitsAndTrees(tracer, enlistment, objectRequestor); @@ -174,11 +182,11 @@ private void PrefetchCommits(ITracer tracer, GVFSEnlistment enlistment, GitObjec { this.ShowStatusWhileRunning( () => { return this.TryPrefetchCommitsAndTrees(tracer, enlistment, objectRequestor); }, - "Fetching commits and trees"); + "Fetching commits and trees " + this.GetCacheServerDisplay(cacheServer)); } } - private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor blobRequestor) + private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor blobRequestor, CacheServerInfo cacheServer) { FetchHelper fetchHelper = new FetchHelper( tracer, @@ -189,10 +197,21 @@ private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjects DownloadThreadCount, IndexThreadCount); - if (!FetchHelper.TryLoadPathWhitelist(tracer, this.PathWhitelist, this.PathWhitelistFile, enlistment, fetchHelper.PathWhitelist)) + string error; + if (!FetchHelper.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, fetchHelper.FolderList, out error)) { - Environment.ExitCode = (int)ReturnCode.GenericError; - return; + this.ReportErrorAndExit(tracer, error); + } + + if (!FetchHelper.TryLoadFileList(enlistment, this.Files, fetchHelper.FileList, out error)) + { + this.ReportErrorAndExit(tracer, error); + } + + if (fetchHelper.FolderList.Count == 0 && + fetchHelper.FileList.Count == 0) + { + this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files *` to confirm."); } GitProcess gitProcess = new GitProcess(enlistment); @@ -205,13 +224,20 @@ private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjects return; } + int matchedBlobs = 0; + int downloadedBlobs = 0; + string headCommitId = result.Output; Func doPrefetch = () => { try { - fetchHelper.FastFetch(headCommitId.Trim(), isBranch: false); + fetchHelper.FastFetchWithStats( + headCommitId.Trim(), + isBranch: false, + matchedBlobs: out matchedBlobs, + downloadedBlobs: out downloadedBlobs); return !fetchHelper.HasFailures; } catch (FetchHelper.FetchException e) @@ -227,18 +253,25 @@ private void PrefetchBlobs(ITracer tracer, GVFSEnlistment enlistment, GitObjects } else { - this.ShowStatusWhileRunning(doPrefetch, "Fetching blobs"); + this.ShowStatusWhileRunning(doPrefetch, "Fetching blobs " + this.GetCacheServerDisplay(cacheServer)); } if (fetchHelper.HasFailures) { Environment.ExitCode = 1; } + else + { + Console.WriteLine("Your filter matched {0} blob(s), and {1} new blob(s) were downloaded.", matchedBlobs, downloadedBlobs); + } } private bool TryPrefetchCommitsAndTrees(ITracer tracer, GVFSEnlistment enlistment, GitObjectsHttpRequestor objectRequestor) { - GitObjects gitObjects = new GitObjects(tracer, enlistment, objectRequestor); + PhysicalFileSystem fileSystem = new PhysicalFileSystem(); + GitRepo repo = new GitRepo(tracer, enlistment, fileSystem); + GVFSContext context = new GVFSContext(tracer, fileSystem, repo, enlistment); + GitObjects gitObjects = new GVFSGitObjects(context, objectRequestor); string[] packs = gitObjects.ReadPackFileNames(GVFSConstants.PrefetchPackPrefix); long max = -1; @@ -272,14 +305,14 @@ private bool TryPrefetchCommitsAndTrees(ITracer tracer, GVFSEnlistment enlistmen return null; } - private static class Parameters + private string GetCacheServerDisplay(CacheServerInfo cacheServer) { - public const string Folders = "folders"; - public const string FoldersList = "folders-list"; - public const string Commits = "commits"; + if (cacheServer.HasResolvedName()) + { + return "from " + cacheServer.Name + " cache server"; + } - public const string DefaultPathWhitelist = ""; - public const string DefaultPathWhitelistFile = ""; + return "from " + cacheServer.Url; } } } diff --git a/GVFS/GVFS/CommandLine/RepairJobs/BackgroundOperationDatabaseRepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/BackgroundOperationDatabaseRepairJob.cs index be279c28a3..685e5ac3a9 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/BackgroundOperationDatabaseRepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/BackgroundOperationDatabaseRepairJob.cs @@ -1,5 +1,7 @@ using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; +using GVFS.GVFlt; using System.Collections.Generic; using System.IO; @@ -7,12 +9,12 @@ namespace GVFS.CommandLine.RepairJobs { public class BackgroundOperationDatabaseRepairJob : RepairJob { - private readonly string databasePath; + private readonly string dataPath; public BackgroundOperationDatabaseRepairJob(ITracer tracer, TextWriter output, GVFSEnlistment enlistment) : base(tracer, output, enlistment) { - this.databasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.BackgroundGitUpdates); + this.dataPath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundGitOperations); } public override string Name @@ -22,8 +24,16 @@ public override string Name public override IssueType HasIssue(List messages) { - if (!this.TryCreatePersistentDictionary(this.databasePath, messages)) + string error; + BackgroundGitUpdateQueue instance; + if (!BackgroundGitUpdateQueue.TryCreate( + this.Tracer, + this.dataPath, + new PhysicalFileSystem(), + out instance, + out error)) { + messages.Add("Failed to read background operations: " + error); return IssueType.CantFix; } diff --git a/GVFS/GVFS/CommandLine/RepairJobs/BlobSizeDatabaseRepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/BlobSizeDatabaseRepairJob.cs index 876db3a5dd..8fd390b08e 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/BlobSizeDatabaseRepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/BlobSizeDatabaseRepairJob.cs @@ -1,5 +1,8 @@ using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; +using Microsoft.Isam.Esent; +using Microsoft.Isam.Esent.Collections.Generic; using System.Collections.Generic; using System.IO; @@ -12,7 +15,7 @@ public class BlobSizeDatabaseRepairJob : RepairJob public BlobSizeDatabaseRepairJob(ITracer tracer, TextWriter output, GVFSEnlistment enlistment) : base(tracer, output, enlistment) { - this.databasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.BlobSizes); + this.databasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.BlobSizesName); } public override string Name @@ -22,8 +25,15 @@ public override string Name public override IssueType HasIssue(List messages) { - if (!this.TryCreatePersistentDictionary(this.databasePath, messages)) + try { + using (PersistentDictionary dict = new PersistentDictionary(this.databasePath)) + { + } + } + catch (EsentException error) + { + messages.Add("Could not load blob size database: " + error); return IssueType.Fixable; } diff --git a/GVFS/GVFS/CommandLine/RepairJobs/GitConfigRepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/GitConfigRepairJob.cs index fead2ec591..4297d1c955 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/GitConfigRepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/GitConfigRepairJob.cs @@ -1,7 +1,6 @@ using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.Tracing; -using System; using System.Collections.Generic; using System.IO; @@ -45,11 +44,19 @@ public override IssueType HasIssue(List messages) // At this point, we've confirmed that the repo url can be gotten, so we have to // reinitialize the GitProcess with a valid repo url for 'git credential fill' - GVFSEnlistment enlistment = GVFSEnlistment.CreateFromDirectory( - this.Enlistment.EnlistmentRoot, - this.Enlistment.GitBinPath, - this.Enlistment.GVFSHooksRoot); - git = new GitProcess(enlistment); + try + { + GVFSEnlistment enlistment = GVFSEnlistment.CreateFromDirectory( + this.Enlistment.EnlistmentRoot, + this.Enlistment.GitBinPath, + this.Enlistment.GVFSHooksRoot); + git = new GitProcess(enlistment); + } + catch (InvalidRepoException) + { + messages.Add("An issue was found that may be a side-effect of other issues. Fix them with 'gvfs repair --confirm' then 'gvfs repair' again."); + return IssueType.CantFix; + } string username; string password; @@ -76,8 +83,7 @@ public override FixResult TryFixIssues(List messages) File.WriteAllText(configPath, string.Empty); this.Tracer.RelatedInfo("Created empty file: " + configPath); - GitProcess git = new GitProcess(this.Enlistment); - if (!GVFSVerb.TrySetGitConfigSettings(git)) + if (!GVFSVerb.TrySetGitConfigSettings(this.Enlistment)) { messages.Add("Unable to create default .git\\config."); diff --git a/GVFS/GVFS/CommandLine/RepairJobs/GitHeadRepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/GitHeadRepairJob.cs index 9602308b71..5f2a2d0e23 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/GitHeadRepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/GitHeadRepairJob.cs @@ -59,8 +59,7 @@ public override FixResult TryFixIssues(List messages) catch (IOException ex) { EventMetadata metadata = new EventMetadata(); - metadata.Add("ErrorMessage", "Failed to write HEAD: " + ex.ToString()); - this.Tracer.RelatedError(metadata); + this.Tracer.RelatedError(metadata, "Failed to write HEAD: " + ex.ToString()); return FixResult.Failure; } diff --git a/GVFS/GVFS/CommandLine/RepairJobs/PlaceholderDatabaseRepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/PlaceholderDatabaseRepairJob.cs index d3a25c6525..f0f3c58ec4 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/PlaceholderDatabaseRepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/PlaceholderDatabaseRepairJob.cs @@ -1,4 +1,5 @@ using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System.Collections.Generic; using System.IO; @@ -12,7 +13,7 @@ public class PlaceholderDatabaseRepairJob : RepairJob public PlaceholderDatabaseRepairJob(ITracer tracer, TextWriter output, GVFSEnlistment enlistment) : base(tracer, output, enlistment) { - this.databasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.PlaceholderList); + this.databasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList); } public override string Name @@ -22,8 +23,16 @@ public override string Name public override IssueType HasIssue(List messages) { - if (!this.TryCreatePersistentDictionary(this.databasePath, messages)) + string error; + PlaceholderListDatabase placeholders; + if (!PlaceholderListDatabase.TryCreate( + this.Tracer, + this.databasePath, + new PhysicalFileSystem(), + out placeholders, + out error)) { + messages.Add(error); return IssueType.CantFix; } diff --git a/GVFS/GVFS/CommandLine/RepairJobs/RepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/RepairJob.cs index d353d535a5..038964137a 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/RepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/RepairJob.cs @@ -3,8 +3,6 @@ using GVFS.Common.Tracing; using GVFS.GVFlt.DotGit; using Microsoft.Diagnostics.Tracing; -using Microsoft.Isam.Esent; -using Microsoft.Isam.Esent.Collections.Generic; using System; using System.Collections.Generic; using System.IO; @@ -110,24 +108,6 @@ protected bool TryDeleteFolder(string filePath) return true; } - protected bool TryCreatePersistentDictionary(string databasePath, List messages) - where KeyType : IComparable - { - try - { - using (new PersistentDictionary(databasePath)) - { - } - - return true; - } - catch (EsentException corruptionEx) - { - messages.Add(corruptionEx.Message); - return false; - } - } - protected IssueType TryParseIndex(string path, List messages) { GVFSContext context = new GVFSContext(this.Tracer, null, null, this.Enlistment); diff --git a/GVFS/GVFS/CommandLine/RepairJobs/RepoMetadataDatabaseRepairJob.cs b/GVFS/GVFS/CommandLine/RepairJobs/RepoMetadataDatabaseRepairJob.cs index d27ab34165..c151d95daf 100644 --- a/GVFS/GVFS/CommandLine/RepairJobs/RepoMetadataDatabaseRepairJob.cs +++ b/GVFS/GVFS/CommandLine/RepairJobs/RepoMetadataDatabaseRepairJob.cs @@ -7,12 +7,9 @@ namespace GVFS.CommandLine.RepairJobs { public class RepoMetadataDatabaseRepairJob : RepairJob { - private readonly string databasePath; - public RepoMetadataDatabaseRepairJob(ITracer tracer, TextWriter output, GVFSEnlistment enlistment) : base(tracer, output, enlistment) { - this.databasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.RepoMetadata); } public override string Name @@ -22,8 +19,10 @@ public override string Name public override IssueType HasIssue(List messages) { - if (!this.TryCreatePersistentDictionary(this.databasePath, messages)) + string error; + if (!RepoMetadata.TryInitialize(this.Tracer, this.Enlistment.DotGVFSRoot, out error)) { + messages.Add("Could not open repo metadata: " + error); return IssueType.CantFix; } diff --git a/GVFS/GVFS/CommandLine/RepairVerb.cs b/GVFS/GVFS/CommandLine/RepairVerb.cs index 6881d69188..b7dd50ecae 100644 --- a/GVFS/GVFS/CommandLine/RepairVerb.cs +++ b/GVFS/GVFS/CommandLine/RepairVerb.cs @@ -1,4 +1,5 @@ using CommandLine; +using GVFS.CommandLine.DiskLayoutUpgrades; using GVFS.CommandLine.RepairJobs; using GVFS.Common; using GVFS.Common.Git; @@ -36,7 +37,7 @@ protected override string VerbName public override void Execute() { - string hooksPath = this.GetGVFSHooksPathAndCheckVersion(); + string hooksPath = this.GetGVFSHooksPathAndCheckVersion(tracer: null); GVFSEnlistment enlistment = GVFSEnlistment.CreateWithoutRepoUrlFromDirectory( this.EnlistmentRootPath, @@ -62,6 +63,12 @@ public override void Execute() "); } + string error; + if (!DiskLayoutUpgrade.TryCheckDiskLayoutVersion(tracer: null, enlistmentRoot: enlistment.EnlistmentRoot, error: out error)) + { + this.ReportErrorAndExit(error); + } + if (!ConsoleHelper.ShowStatusWhileRunning( () => { @@ -80,7 +87,7 @@ public override void Execute() "Checking if GVFS is mounted", this.Output, showSpinner: true, - gvfsLogEnlistmentRoot: enlistment.EnlistmentRoot)) + gvfsLogEnlistmentRoot: null)) { this.ReportErrorAndExit("You can only run 'gvfs repair' if GVFS is not mounted. Run 'gvfs unmount' and try again."); } @@ -97,19 +104,22 @@ public override void Execute() enlistment.EnlistmentRoot, enlistment.RepoUrl, "N/A", + enlistment.GitObjectsRoot, new EventMetadata { - { "Confirmed", this.Confirmed } + { "Confirmed", this.Confirmed }, + { "IsElevated", ProcessHelper.IsAdminElevated() }, }); List jobs = new List(); - - // Repair ESENT Databases + + // Repair databases jobs.Add(new BackgroundOperationDatabaseRepairJob(tracer, this.Output, enlistment)); - jobs.Add(new BlobSizeDatabaseRepairJob(tracer, this.Output, enlistment)); - jobs.Add(new PlaceholderDatabaseRepairJob(tracer, this.Output, enlistment)); jobs.Add(new RepoMetadataDatabaseRepairJob(tracer, this.Output, enlistment)); + jobs.Add(new PlaceholderDatabaseRepairJob(tracer, this.Output, enlistment)); + jobs.Add(new BlobSizeDatabaseRepairJob(tracer, this.Output, enlistment)); + // Repair .git folder files jobs.Add(new GitHeadRepairJob(tracer, this.Output, enlistment)); jobs.Add(new GitIndexRepairJob(tracer, this.Output, enlistment)); jobs.Add(new GitConfigRepairJob(tracer, this.Output, enlistment)); @@ -193,7 +203,7 @@ public override void Execute() private void WriteMessage(ITracer tracer, string message) { - tracer.RelatedEvent(EventLevel.Informational, "RepairInfo", new EventMetadata { { "Message", message } }); + tracer.RelatedEvent(EventLevel.Informational, "RepairInfo", new EventMetadata { { TracingConstants.MessageKey.InfoMessage, message } }); this.Output.WriteLine(message); } diff --git a/GVFS/GVFS/CommandLine/UnmountVerb.cs b/GVFS/GVFS/CommandLine/UnmountVerb.cs index 4572fa0ecf..e2d47ac729 100644 --- a/GVFS/GVFS/CommandLine/UnmountVerb.cs +++ b/GVFS/GVFS/CommandLine/UnmountVerb.cs @@ -53,11 +53,14 @@ public override void Execute() this.ReportErrorAndExit(errorMessage); } - if (!this.ShowStatusWhileRunning( - () => { return this.UnregisterRepo(root, out errorMessage); }, - "Unregistering automount")) + if (!this.Unattended) { - this.Output.WriteLine(" WARNING: " + errorMessage); + if (!this.ShowStatusWhileRunning( + () => { return this.UnregisterRepo(root, out errorMessage); }, + "Unregistering automount")) + { + this.Output.WriteLine(" WARNING: " + errorMessage); + } } } @@ -153,7 +156,7 @@ private bool UnregisterRepo(string rootPath, out string errorMessage) { if (!client.Connect()) { - errorMessage = "Unable to unregister repo because GVFS.Service is not responding. Run 'sc start GVFS.Service' from an elevated command prompt to ensure it is running."; + errorMessage = "Unable to unregister repo because GVFS.Service is not responding. " + GVFSVerb.StartServiceInstructions; return false; } @@ -206,9 +209,11 @@ private void AcquireLock(string enlistmentRoot) Process currentProcess = Process.GetCurrentProcess(); string result = null; if (!GVFSLock.TryAcquireGVFSLockForProcess( + this.Unattended, pipeClient, "gvfs unmount", currentProcess.Id, + ProcessHelper.IsAdminElevated(), currentProcess, enlistmentRoot, out result)) diff --git a/GVFS/GVFS/GVFS.csproj b/GVFS/GVFS/GVFS.csproj index 64fcfd7eaf..dbbd4f1b27 100644 --- a/GVFS/GVFS/GVFS.csproj +++ b/GVFS/GVFS/GVFS.csproj @@ -41,8 +41,9 @@ true - - ..\..\..\packages\CommandLineParser.2.0.275-beta\lib\net45\CommandLine.dll + + False + ..\..\..\packages\CommandLineParser.2.1.1-beta\lib\net45\CommandLine.dll True @@ -85,6 +86,9 @@ + + + @@ -101,6 +105,8 @@ + + @@ -123,9 +129,9 @@ {374bf1e5-0b2d-4d4a-bd5e-4212299def09} GVFS.Common - + {fb0831ae-9997-401b-b31f-3a065fdbeb20} - GvFlt + GvLib.Managed {1118b427-7063-422f-83b9-5023c8ec5a7a} @@ -150,8 +156,8 @@ xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.ReadObjectHook\bin\$(Platform)\$(Configuration)\GVFS.ReadObjectHook.* $(TargetDir) xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.Hooks\bin\$(Platform)\$(Configuration)\GVFS.Hooks.* $(TargetDir) -xcopy /Y $(SolutionDir)..\BuildOutput\GitHooksLoader\bin\$(Platform)\$(Configuration)\GitHooksLoader.* $(TargetDir) -cmd /c ""c:\Program Files (x86)\Inno Setup 5\ISCC.exe" "/DPlatformAndConfiguration=$(Platform)\$(Configuration)" /DGvFltPackage="$(GvFltPackage)" $(TargetDir)Setup.iss" +xcopy /Y $(SolutionDir)..\BuildOutput\GitHooksLoader\bin\$(Platform)\$(Configuration)\GitHooksLoader.* $(TargetDir) +cmd /c ""c:\Program Files (x86)\Inno Setup 5\ISCC.exe" /DPlatformAndConfiguration="$(Platform)\$(Configuration)" /DGvFltPackage="$(GvFltPackage)" $(TargetDir)Setup.iss" diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index 503a714f93..c1b02a4bb2 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -1,6 +1,8 @@ using CommandLine; using GVFS.CommandLine; +using GVFS.Common; using System; +using System.Linq; // This is to keep the reference to GVFS.Mount // so that the exe will end up in the output directory of GVFS @@ -40,12 +42,21 @@ public static void Main(string[] args) settings.HelpWriter = Console.Error; }) .ParseArguments(args, verbTypes) + .WithNotParsed( + errors => + { + if (errors.Any(error => error is TokenError)) + { + Environment.ExitCode = (int)ReturnCode.ParsingError; + } + }) .WithParsed( clone => { // We handle the clone verb differently, because clone cares if the enlistment path // was not specified vs if it was specified to be the current directory clone.Execute(); + Environment.ExitCode = (int)ReturnCode.Success; }) .WithParsed( verb => @@ -58,6 +69,7 @@ public static void Main(string[] args) } verb.Execute(); + Environment.ExitCode = (int)ReturnCode.Success; }); } catch (GVFSVerb.VerbAbortedException e) diff --git a/GVFS/GVFS/Setup.iss b/GVFS/GVFS/Setup.iss index 51898d332c..c0c16f222b 100644 --- a/GVFS/GVFS/Setup.iss +++ b/GVFS/GVFS/Setup.iss @@ -10,7 +10,6 @@ #define MyAppURL "https://github.com/Microsoft/gvfs" #define MyAppExeName "GVFS.exe" #define EnvironmentKey "SYSTEM\CurrentControlSet\Control\Session Manager\Environment" -#define GvFltParametersKey "SYSTEM\CurrentControlSet\Services\Gvflt\Parameters" #define GVFltRelative "..\..\..\..\..\packages\" + GvFltPackage + "\filter" #define GVFSCommonRelative "..\..\..\..\GVFS.Common\bin" @@ -109,7 +108,7 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"Esent.Interop.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"Esent.Isam.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.Common.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.GVFlt.dll" -DestDir: "{app}"; Flags: ignoreversion; Source:"GvFlt.dll" +DestDir: "{app}"; Flags: ignoreversion; Source:"GvLib.Managed.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"GvLib.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"Microsoft.Diagnostics.Tracing.EventSource.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"Newtonsoft.Json.dll" @@ -282,10 +281,7 @@ begin // Note: Programatic install of INF notifies user if the driver being upgraded to is older than the existing, otherwise it works silently... doesn't seem like there is a way to block if Exec(ExpandConstant('RUNDLL32.EXE'), ExpandConstant('SETUPAPI.DLL,InstallHinfSection DefaultInstall 128 {app}\Filter\gvflt.inf'), '', SW_HIDE, ewWaitUntilTerminated, ResultCode) then begin - if RegWriteDWordValue(HKEY_LOCAL_MACHINE, '{#GvFltParametersKey}', 'CommandTimeoutInMs', 86400000) then - begin - InstallSuccessful := True; - end; + InstallSuccessful := True; end; finally WizardForm.StatusLabel.Caption := StatusText; diff --git a/GVFS/GVFS/packages.config b/GVFS/GVFS/packages.config index dfaee1986a..b1fab75a49 100644 --- a/GVFS/GVFS/packages.config +++ b/GVFS/GVFS/packages.config @@ -1,6 +1,6 @@  - + diff --git a/GitHooksLoader/GitHooksLoader.cpp b/GitHooksLoader/GitHooksLoader.cpp index 0e1c0a91f0..d86638ef86 100644 --- a/GitHooksLoader/GitHooksLoader.cpp +++ b/GitHooksLoader/GitHooksLoader.cpp @@ -9,8 +9,8 @@ int ExecuteHook(const std::wstring &applicationName, wchar_t *hookName, int argc int wmain(int argc, WCHAR *argv[]) { - LARGE_INTEGER tickFrequency; - LARGE_INTEGER startTime, endTime; + LARGE_INTEGER tickFrequency = { 0 }; + LARGE_INTEGER startTime = { 0 }, endTime = { 0 }; bool perfTraceEnabled = false; size_t requiredCount = 0; diff --git a/GitHooksLoader/GitHooksLoader.vcxproj b/GitHooksLoader/GitHooksLoader.vcxproj index a9c806b9b0..edc22633ed 100644 --- a/GitHooksLoader/GitHooksLoader.vcxproj +++ b/GitHooksLoader/GitHooksLoader.vcxproj @@ -14,7 +14,7 @@ {798DE293-6EDA-4DC4-9395-BE7A71C563E3} Win32Proj GitHooksLoader - 8.1 + 10.0.10240.0 @@ -44,51 +44,64 @@ true - $(SolutionDir)\..\BuildOutput\$(ProjectName)\bin\$(Platform)\$(Configuration)\ - $(SolutionDir)\..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\ + $(SolutionDir)..\BuildOutput\$(ProjectName)\bin\$(Platform)\$(Configuration)\ + $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\ false - $(SolutionDir)\..\BuildOutput\$(ProjectName)\bin\$(Platform)\$(Configuration)\ - $(SolutionDir)\..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\ + $(SolutionDir)..\BuildOutput\$(ProjectName)\bin\$(Platform)\$(Configuration)\ + $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\ Use - Level3 + Level4 Disabled _DEBUG;_CONSOLE;%(PreprocessorDefinitions) true + true + C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;%(AdditionalIncludeDirectories) Console true + C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;%(AdditionalLibraryDirectories) - $(SolutionDir)\..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log + $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log + + $(SolutionDir)\..\BuildOutput + - Level3 + Level4 Use MaxSpeed true true NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true + true + C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;%(AdditionalIncludeDirectories) Console true true true + C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;%(AdditionalLibraryDirectories) - $(SolutionDir)\..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log + $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log + + $(SolutionDir)\..\BuildOutput + + @@ -99,6 +112,9 @@ Create + + + diff --git a/GitHooksLoader/GitHooksLoader.vcxproj.filters b/GitHooksLoader/GitHooksLoader.vcxproj.filters index 2f86b6c7b7..336e774530 100644 --- a/GitHooksLoader/GitHooksLoader.vcxproj.filters +++ b/GitHooksLoader/GitHooksLoader.vcxproj.filters @@ -21,6 +21,9 @@ Header Files + + Header Files + @@ -30,4 +33,9 @@ Source Files + + + Resource Files + + \ No newline at end of file diff --git a/GitHooksLoader/Version.rc b/GitHooksLoader/Version.rc new file mode 100644 index 0000000000..b9c49f3183 Binary files /dev/null and b/GitHooksLoader/Version.rc differ diff --git a/GitHooksLoader/resource.h b/GitHooksLoader/resource.h new file mode 100644 index 0000000000..7169f3908d Binary files /dev/null and b/GitHooksLoader/resource.h differ diff --git a/Protocol.md b/Protocol.md index a172957d58..dafa77db7f 100644 --- a/Protocol.md +++ b/Protocol.md @@ -194,34 +194,26 @@ An example response is provided below. Note that the `null` `"Max"` value is onl "Major": 0, "Minor": 4, "Build": 0, - "Revision": 0, - "MajorRevision": 0, - "MinorRevision": 0 + "Revision": 0 }, "Min": { "Major": 0, "Minor": 2, "Build": 0, - "Revision": 0, - "MajorRevision": 0, - "MinorRevision": 0 + "Revision": 0 } }, { "Max": { "Major": 0, "Minor": 5, "Build": 0, - "Revision": 0, - "MajorRevision": 0, - "MinorRevision": 0 + "Revision": 0 }, "Min": { "Major": 0, "Minor": 4, "Build": 17009, - "Revision": 1, - "MajorRevision": 0, - "MinorRevision": 1 + "Revision": 1 } }, { "Max": null, @@ -229,9 +221,7 @@ An example response is provided below. Note that the `null` `"Max"` value is onl "Major": 0, "Minor": 5, "Build": 16326, - "Revision": 1, - "MajorRevision": 0, - "MinorRevision": 1 + "Revision": 1 } }], "CacheServers": [{ diff --git a/Scripts/CreateCommonAssemblyVersion.bat b/Scripts/CreateCommonAssemblyVersion.bat deleted file mode 100644 index b13562c703..0000000000 --- a/Scripts/CreateCommonAssemblyVersion.bat +++ /dev/null @@ -1,2 +0,0 @@ -mkdir %2\BuildOutput -echo using System.Reflection; [assembly: AssemblyVersion("%1")][assembly: AssemblyFileVersion("%1")] > %2\BuildOutput\CommonAssemblyVersion.cs \ No newline at end of file diff --git a/Scripts/CreateCommonCliAssemblyVersion.bat b/Scripts/CreateCommonCliAssemblyVersion.bat deleted file mode 100644 index 48a7db4d29..0000000000 --- a/Scripts/CreateCommonCliAssemblyVersion.bat +++ /dev/null @@ -1,3 +0,0 @@ -mkdir %2\BuildOutput -echo #include "stdafx.h" > %2\BuildOutput\CommonAssemblyVersion.h -echo using namespace System::Reflection; [assembly:AssemblyVersion("%1")];[assembly:AssemblyFileVersion("%1")]; >> %2\BuildOutput\CommonAssemblyVersion.h \ No newline at end of file diff --git a/Scripts/CreateCommonVersionHeader.bat b/Scripts/CreateCommonVersionHeader.bat deleted file mode 100644 index 0621478bdc..0000000000 --- a/Scripts/CreateCommonVersionHeader.bat +++ /dev/null @@ -1,9 +0,0 @@ -mkdir %2\BuildOutput - -set comma_version_string=%1 -set comma_version_string=%comma_version_string:.=,% - -echo #define GVFS_FILE_VERSION %comma_version_string% > %2\BuildOutput\CommonVersionHeader.h -echo #define GVFS_FILE_VERSION_STRING "%1" >> %2\BuildOutput\CommonVersionHeader.h -echo #define GVFS_PRODUCT_VERSION %comma_version_string% >> %2\BuildOutput\CommonVersionHeader.h -echo #define GVFS_PRODUCT_VERSION_STRING "%1" >> %2\BuildOutput\CommonVersionHeader.h \ No newline at end of file