diff --git a/GVFS.sln b/GVFS.sln index e9bcd4ea79..1a8794c4d9 100644 --- a/GVFS.sln +++ b/GVFS.sln @@ -22,7 +22,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "GVFS", "GVFS", "{2EF2EC94-3 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.GVFlt", "GVFS\GVFS.GVFlt\GVFS.GVFlt.csproj", "{1118B427-7063-422F-83B9-5023C8EC5A7A}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GVFS.GvFltWrapper", "GVFS\GVFS.GvFltWrapper\GVFS.GvFltWrapper.vcxproj", "{FB0831AE-9997-401B-B31F-3A065FDBEB20}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GvFlt", "GVFS\GVFS.GvFltWrapper\GvFlt.vcxproj", "{FB0831AE-9997-401B-B31F-3A065FDBEB20}" ProjectSection(ProjectDependencies) = postProject {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258} {374BF1E5-0B2D-4D4A-BD5E-4212299DEF09} = {374BF1E5-0B2D-4D4A-BD5E-4212299DEF09} @@ -61,10 +61,15 @@ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Hooks", "GVFS\GVFS.Hooks\GVFS.Hooks.csproj", "{BDA91EE5-C684-4FC5-A90A-B7D677421917}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Service", "GVFS\GVFS.Service\GVFS.Service.csproj", "{B8C1DFBA-CAFD-4F7E-A1A3-E11907B5467B}" + ProjectSection(ProjectDependencies) = postProject + {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258} + {BDA91EE5-C684-4FC5-A90A-B7D677421917} = {BDA91EE5-C684-4FC5-A90A-B7D677421917} + EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Mount", "GVFS\GVFS.Mount\GVFS.Mount.csproj", "{17498502-AEFF-4E70-90CC-1D0B56A8ADF5}" ProjectSection(ProjectDependencies) = postProject {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258} + {BDA91EE5-C684-4FC5-A90A-B7D677421917} = {BDA91EE5-C684-4FC5-A90A-B7D677421917} EndProjectSection EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GVFS.ReadObjectHook", "GVFS\GVFS.ReadObjectHook\GVFS.ReadObjectHook.vcxproj", "{5A6656D5-81C7-472C-9DC8-32D071CB2258}" diff --git a/GVFS/FastFetch/CheckoutFetchHelper.cs b/GVFS/FastFetch/CheckoutFetchHelper.cs index 3b3e25fb93..c354c181a9 100644 --- a/GVFS/FastFetch/CheckoutFetchHelper.cs +++ b/GVFS/FastFetch/CheckoutFetchHelper.cs @@ -2,10 +2,12 @@ using FastFetch.Jobs; using GVFS.Common; using GVFS.Common.Git; +using GVFS.Common.Http; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; using System.Collections.Generic; +using System.IO; using System.Linq; namespace FastFetch @@ -18,12 +20,13 @@ public class CheckoutFetchHelper : FetchHelper public CheckoutFetchHelper( ITracer tracer, Enlistment enlistment, + GitObjectsHttpRequestor objectRequestor, int chunkSize, int searchThreadCount, int downloadThreadCount, int indexThreadCount, int checkoutThreadCount, - bool allowIndexMetadataUpdateFromWorkingTree) : base(tracer, enlistment, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount) + bool allowIndexMetadataUpdateFromWorkingTree) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount) { this.checkoutThreadCount = checkoutThreadCount; this.allowIndexMetadataUpdateFromWorkingTree = allowIndexMetadataUpdateFromWorkingTree; @@ -51,7 +54,7 @@ public override void FastFetch(string branchOrCommit, bool isBranch) throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } - commitToFetch = refs.GetTipCommitIds().Single(); + commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { @@ -98,7 +101,7 @@ public override void FastFetch(string branchOrCommit, bool isBranch) if (isBranch) { // Update the refspec before setting the upstream or git will complain the remote branch doesn't exist - this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); using (ITracer activity = this.Tracer.StartActivity("SetUpstream", EventLevel.Informational)) { @@ -113,27 +116,30 @@ public override void FastFetch(string branchOrCommit, bool isBranch) } } - bool indexSigningIsOff = this.GetIsIndexSigningOff(); + bool shouldSignIndex = !this.GetIsIndexSigningOff(); // Update the index EventMetadata updateIndexMetadata = new EventMetadata(); - updateIndexMetadata.Add("IndexSigningIsOff", indexSigningIsOff); + updateIndexMetadata.Add("IndexSigningIsOff", shouldSignIndex); using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata)) { - // Create the index object now so it can track the current index - Index index = indexSigningIsOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null; - - GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, !indexSigningIsOff); - indexGen.CreateFromHeadTree(); - this.HasFailures = indexGen.HasFailures; - - if (!indexGen.HasFailures && index != null) + Index sourceIndex = this.GetSourceIndex(); + GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, shouldSignIndex); + indexGen.CreateFromHeadTree(indexVersion: 2); + this.HasFailures |= indexGen.HasFailures; + + if (!indexGen.HasFailures) { + Index newIndex = new Index( + this.Enlistment.EnlistmentRoot, + this.Tracer, + Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName), + readOnly: false); + // Update from disk only if the caller says it is ok via command line // or if we updated the whole tree and know that all files are up to date bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree; - - index.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree); + newIndex.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree, shouldSignIndex, sourceIndex); } } } @@ -146,24 +152,44 @@ public override void FastFetch(string branchOrCommit, bool isBranch) /// protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs) { - UpdateRefsHelper refHelper = new UpdateRefsHelper(this.Enlistment); - if (isBranch) { KeyValuePair remoteRef = refs.GetBranchRefPairs().Single(); string remoteBranch = remoteRef.Key; - string fullLocalBranchName = branchOrCommit.StartsWith("refs/heads/") ? branchOrCommit : ("refs/heads/" + branchOrCommit); - this.HasFailures |= !refHelper.UpdateRef(this.Tracer, fullLocalBranchName, remoteRef.Value); - this.HasFailures |= !refHelper.UpdateRef(this.Tracer, "HEAD", fullLocalBranchName); + string fullLocalBranchName = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit); + this.HasFailures |= !this.UpdateRef(this.Tracer, fullLocalBranchName, remoteRef.Value); + this.HasFailures |= !this.UpdateRef(this.Tracer, "HEAD", fullLocalBranchName); } else { - this.HasFailures |= !refHelper.UpdateRef(this.Tracer, "HEAD", branchOrCommit); + this.HasFailures |= !this.UpdateRef(this.Tracer, "HEAD", branchOrCommit); } base.UpdateRefs(branchOrCommit, isBranch, refs); } + + private Index GetSourceIndex() + { + string indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); + string backupIndexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName + ".backup"); + + if (File.Exists(indexPath)) + { + // Note that this moves the current index, leaving nothing behind + // This is intentional as we only need it for the purpose of updating the + // new index and leaving it behind can make updating slower. + this.Tracer.RelatedEvent(EventLevel.Informational, "CreateBackup", new EventMetadata() { { "BackupIndexName", backupIndexPath } }); + File.Delete(backupIndexPath); + File.Move(indexPath, backupIndexPath); + + Index output = new Index(this.Enlistment.EnlistmentRoot, this.Tracer, backupIndexPath, readOnly: true); + output.Parse(); + return output; + } + + return null; + } private bool GetIsIndexSigningOff() { diff --git a/GVFS/FastFetch/FastFetch.csproj b/GVFS/FastFetch/FastFetch.csproj index 86d5321b5b..f1966e5856 100644 --- a/GVFS/FastFetch/FastFetch.csproj +++ b/GVFS/FastFetch/FastFetch.csproj @@ -67,13 +67,14 @@ + + - @@ -83,7 +84,6 @@ - diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index 9b71930c8f..c32358f31a 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -1,6 +1,7 @@ using CommandLine; using GVFS.Common; using GVFS.Common.Git; +using GVFS.Common.Http; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; @@ -87,9 +88,9 @@ public class FastFetchVerb "max-retries", Required = false, Default = 10, - HelpText = "Sets the maximum number of retries for downloading a pack")] + HelpText = "Sets the maximum number of attempts for downloading a pack")] - public int MaxRetries { get; set; } + public int MaxAttempts { get; set; } [Option( "git-path", @@ -157,9 +158,7 @@ private int ExecuteWithExitCode() Console.WriteLine("Cannot specify both a commit sha and a branch name."); return ExitFailure; } - - this.CacheServerUrl = Enlistment.StripObjectsEndpointSuffix(this.CacheServerUrl); - + this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount; this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads); this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount; @@ -167,7 +166,7 @@ private int ExecuteWithExitCode() this.GitBinPath = !string.IsNullOrWhiteSpace(this.GitBinPath) ? this.GitBinPath : GitProcess.GetInstalledGitBinPath(); - GitEnlistment enlistment = GitEnlistment.CreateFromCurrentDirectory(this.CacheServerUrl, this.GitBinPath); + GitEnlistment enlistment = GitEnlistment.CreateFromCurrentDirectory(this.GitBinPath); if (enlistment == null) { Console.WriteLine("Must be run within a git repo"); @@ -206,19 +205,28 @@ private int ExecuteWithExitCode() string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch"); tracer.AddLogFileEventListener(fastfetchLogFile, EventLevel.Informational, Keywords.Any); + + RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); + + string error; + CacheServerInfo cacheServer; + if (!CacheServerInfo.TryDetermineCacheServer(this.CacheServerUrl, tracer, enlistment, retryConfig, out cacheServer, out error)) + { + tracer.RelatedError(error); + return ExitFailure; + } + tracer.WriteStartEvent( enlistment.EnlistmentRoot, enlistment.RepoUrl, - enlistment.CacheServerUrl, + cacheServer.Url, new EventMetadata { { "TargetCommitish", commitish }, { "Checkout", this.Checkout }, }); - - FetchHelper fetchHelper = this.GetFetchHelper(tracer, enlistment); - fetchHelper.MaxRetries = this.MaxRetries; - + + FetchHelper fetchHelper = this.GetFetchHelper(tracer, enlistment, cacheServer, retryConfig); if (!FetchHelper.TryLoadPathWhitelist(tracer, this.PathWhitelist, this.PathWhitelistFile, enlistment, fetchHelper.PathWhitelist)) { return ExitFailure; @@ -254,7 +262,7 @@ private int ExecuteWithExitCode() "Fetching", output: Console.Out, showSpinner: !Console.IsOutputRedirected, - suppressGvfsLogMessage: true); + gvfsLogEnlistmentRoot: null); Console.WriteLine(); Console.WriteLine("See the full log at " + fastfetchLogFile); @@ -284,13 +292,16 @@ private int ExecuteWithExitCode() } } - private FetchHelper GetFetchHelper(ITracer tracer, Enlistment enlistment) + private FetchHelper GetFetchHelper(ITracer tracer, Enlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig) { + GitObjectsHttpRequestor objectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, cacheServer, retryConfig); + if (this.Checkout) { return new CheckoutFetchHelper( tracer, enlistment, + objectRequestor, this.ChunkSize, this.SearchThreadCount, this.DownloadThreadCount, @@ -303,6 +314,7 @@ private FetchHelper GetFetchHelper(ITracer tracer, Enlistment enlistment) return new FetchHelper( tracer, enlistment, + objectRequestor, this.ChunkSize, this.SearchThreadCount, this.DownloadThreadCount, diff --git a/GVFS/FastFetch/FetchHelper.cs b/GVFS/FastFetch/FetchHelper.cs index 86cce32c53..61e2b09c16 100644 --- a/GVFS/FastFetch/FetchHelper.cs +++ b/GVFS/FastFetch/FetchHelper.cs @@ -1,5 +1,4 @@ -using FastFetch.Git; -using FastFetch.Jobs; +using FastFetch.Jobs; using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.Http; @@ -15,6 +14,8 @@ namespace FastFetch { public class FetchHelper { + protected const string RefsHeadsGitPath = "refs/heads/"; + protected readonly Enlistment Enlistment; protected readonly GitObjectsHttpRequestor ObjectRequestor; protected readonly GitObjects GitObjects; @@ -28,13 +29,12 @@ public class FetchHelper protected readonly bool SkipConfigUpdate; private const string AreaPath = nameof(FetchHelper); - - // Shallow clones don't require their parent commits private const int CommitDepth = 1; public FetchHelper( ITracer tracer, Enlistment enlistment, + GitObjectsHttpRequestor objectRequestor, int chunkSize, int searchThreadCount, int downloadThreadCount, @@ -46,7 +46,7 @@ public class FetchHelper this.ChunkSize = chunkSize; this.Tracer = tracer; this.Enlistment = enlistment; - this.ObjectRequestor = new GitObjectsHttpRequestor(tracer, enlistment); + this.ObjectRequestor = objectRequestor; this.GitObjects = new GitObjects(tracer, enlistment, this.ObjectRequestor); this.PathWhitelist = new List(); @@ -54,12 +54,6 @@ public class FetchHelper this.SkipConfigUpdate = enlistment is GVFSEnlistment; } - public int MaxRetries - { - get { return this.ObjectRequestor.MaxRetries; } - set { this.ObjectRequestor.MaxRetries = value; } - } - public bool HasFailures { get; protected set; } public List PathWhitelist { get; private set; } @@ -116,7 +110,7 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch) throw new FetchException("Could not find branch {0} in info/refs from: {1}", branchOrCommit, this.Enlistment.RepoUrl); } - commitToFetch = refs.GetTipCommitIds().Single(); + commitToFetch = refs.GetTipCommitId(branchOrCommit); } else { @@ -176,9 +170,36 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch) if (isBranch) { - this.HasFailures |= !RefSpecHelpers.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + this.HasFailures |= !this.UpdateRefSpec(this.Tracer, this.Enlistment, branchOrCommit, refs); + } + } + } + + protected bool UpdateRefSpec(ITracer tracer, Enlistment enlistment, string branchOrCommit, GitRefs refs) + { + using (ITracer activity = tracer.StartActivity("UpdateRefSpec", EventLevel.Informational, Keywords.Telemetry, metadata: null)) + { + const string OriginRefMapSettingName = "remote.origin.fetch"; + + // We must update the refspec to get proper "git pull" functionality. + string localBranch = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit); + string remoteBranch = refs.GetBranchRefPairs().Single().Key; + string refSpec = "+" + localBranch + ":" + remoteBranch; + + GitProcess git = new GitProcess(enlistment); + + // Replace all ref-specs this + // * ensures the default refspec (remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*) is removed which avoids some "git fetch/pull" failures + // * gives added "git fetch" performance since git will only fetch the branch provided in the refspec. + GitProcess.Result setResult = git.SetInLocalConfig(OriginRefMapSettingName, refSpec, replaceAll: true); + if (setResult.HasErrors) + { + activity.RelatedError("Could not update ref spec to {0}: {1}", refSpec, setResult.Errors); + return false; } } + + return true; } /// @@ -187,7 +208,6 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch) /// protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs refs) { - UpdateRefsHelper refHelper = new UpdateRefsHelper(this.Enlistment); string commitSha = null; if (isBranch) { @@ -195,7 +215,7 @@ protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs string remoteBranch = remoteRef.Key; commitSha = remoteRef.Value; - this.HasFailures |= !refHelper.UpdateRef(this.Tracer, remoteBranch, commitSha); + this.HasFailures |= !this.UpdateRef(this.Tracer, remoteBranch, commitSha); } else { @@ -206,6 +226,35 @@ protected virtual void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs File.AppendAllText(Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Shallow), commitSha + "\n"); } + protected bool UpdateRef(ITracer tracer, string refName, string targetCommitish) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("RefName", refName); + metadata.Add("TargetCommitish", targetCommitish); + using (ITracer activity = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata)) + { + GitProcess gitProcess = new GitProcess(this.Enlistment); + GitProcess.Result result = null; + if (this.IsSymbolicRef(targetCommitish)) + { + // Using update-ref with a branch name will leave a SHA in the ref file which detaches HEAD, so use symbolic-ref instead. + result = gitProcess.UpdateBranchSymbolicRef(refName, targetCommitish); + } + else + { + result = gitProcess.UpdateBranchSha(refName, targetCommitish); + } + + if (result.HasErrors) + { + activity.RelatedError(result.Errors); + return false; + } + + return true; + } + } + protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects) { EventMetadata startMetadata = new EventMetadata(); @@ -218,18 +267,23 @@ protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects) { if (!repo.ObjectExists(commitSha)) { - if (!gitObjects.TryDownloadAndSaveCommits(new[] { commitSha }, commitDepth: CommitDepth)) + if (!gitObjects.TryDownloadAndSaveCommit(commitSha, commitDepth: CommitDepth)) { EventMetadata metadata = new EventMetadata(); - metadata.Add("ObjectsEndpointUrl", this.Enlistment.ObjectsEndpointUrl); + metadata.Add("ObjectsEndpointUrl", this.ObjectRequestor.CacheServer.ObjectsEndpointUrl); activity.RelatedError(metadata); - throw new FetchException("Could not download commits from {0}", this.Enlistment.ObjectsEndpointUrl); + throw new FetchException("Could not download commits from {0}", this.ObjectRequestor.CacheServer.ObjectsEndpointUrl); } } } } } + private bool IsSymbolicRef(string targetCommitish) + { + return targetCommitish.StartsWith("refs/", StringComparison.OrdinalIgnoreCase); + } + public class FetchException : Exception { public FetchException(string format, params object[] args) diff --git a/GVFS/GVFS.Common/Physical/Git/BigEndianReader.cs b/GVFS/FastFetch/Git/BigEndianReader.cs similarity index 92% rename from GVFS/GVFS.Common/Physical/Git/BigEndianReader.cs rename to GVFS/FastFetch/Git/BigEndianReader.cs index e247d30e82..063167e63b 100644 --- a/GVFS/GVFS.Common/Physical/Git/BigEndianReader.cs +++ b/GVFS/FastFetch/Git/BigEndianReader.cs @@ -1,7 +1,7 @@ using System.IO; using System.Text; -namespace GVFS.Common.Physical.Git +namespace FastFetch.Git { public class BigEndianReader : BinaryReader { diff --git a/GVFS/FastFetch/Git/DiffHelper.cs b/GVFS/FastFetch/Git/DiffHelper.cs index b88f5c60cf..6b123bf6f3 100644 --- a/GVFS/FastFetch/Git/DiffHelper.cs +++ b/GVFS/FastFetch/Git/DiffHelper.cs @@ -20,12 +20,19 @@ public class DiffHelper private HashSet stagedFileDeletes = new HashSet(StringComparer.OrdinalIgnoreCase); private Enlistment enlistment; + private GitProcess git; public DiffHelper(ITracer tracer, Enlistment enlistment, IEnumerable pathWhitelist) + : this(tracer, enlistment, new GitProcess(enlistment), pathWhitelist) + { + } + + public DiffHelper(ITracer tracer, Enlistment enlistment, GitProcess git, IEnumerable pathWhitelist) { this.tracer = tracer; this.pathWhitelist = new List(pathWhitelist); this.enlistment = enlistment; + this.git = git; this.DirectoryOperations = new ConcurrentQueue(); this.FileDeleteOperations = new ConcurrentQueue(); @@ -84,28 +91,42 @@ public void PerformDiff(string sourceTreeSha, string targetTreeSha) metadata.Add("HeadTreeSha", sourceTreeSha); using (ITracer activity = this.tracer.StartActivity("PerformDiff", EventLevel.Informational, Keywords.Telemetry, metadata)) { - GitProcess git = new GitProcess(this.enlistment); - metadata = new EventMetadata(); if (sourceTreeSha == null) { this.UpdatedWholeTree = true; // Nothing is checked out (fresh git init), so we must search the entire tree. - git.LsTree( + GitProcess.Result result = this.git.LsTree( targetTreeSha, line => this.EnqueueOperationsFromLsTreeLine(activity, line), recursive: true, showAllTrees: true); + + if (result.HasErrors) + { + this.HasFailures = true; + metadata.Add("Errors", result.Errors); + metadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output); + } + metadata.Add("Operation", "LsTree"); } else { // Diff head and target, determine what needs to be done. - git.DiffTree( + GitProcess.Result result = this.git.DiffTree( sourceTreeSha, targetTreeSha, line => this.EnqueueOperationsFromDiffTreeLine(this.tracer, this.enlistment.EnlistmentRoot, line)); + + if (result.HasErrors) + { + this.HasFailures = true; + metadata.Add("Errors", result.Errors); + metadata.Add("Output", result.Output.Length > 1024 ? result.Output.Substring(1024) : result.Output); + } + metadata.Add("Operation", "DiffTree"); } diff --git a/GVFS/FastFetch/Git/EndianHelper.cs b/GVFS/FastFetch/Git/EndianHelper.cs new file mode 100644 index 0000000000..5bdd7d766f --- /dev/null +++ b/GVFS/FastFetch/Git/EndianHelper.cs @@ -0,0 +1,48 @@ +namespace FastFetch.Git +{ + public static class EndianHelper + { + public static short Swap(short source) + { + return (short)Swap((ushort)source); + } + + public static int Swap(int source) + { + return (int)Swap((uint)source); + } + + public static long Swap(long source) + { + return (long)((ulong)source); + } + + public static ushort Swap(ushort source) + { + return (ushort)(((source & 0x000000FF) << 8) | + ((source & 0x0000FF00) >> 8)); + } + + public static uint Swap(uint source) + { + return + ((source & 0x000000FF) << 24) | + ((source & 0x0000FF00) << 8) | + ((source & 0x00FF0000) >> 8) | + ((source & 0xFF000000) >> 24); + } + + public static ulong Swap(ulong source) + { + return + ((source & 0x00000000000000FF) << 56) | + ((source & 0x000000000000FF00) << 40) | + ((source & 0x0000000000FF0000) << 24) | + ((source & 0x00000000FF000000) << 8) | + ((source & 0x000000FF00000000) >> 8) | + ((source & 0x0000FF0000000000) >> 24) | + ((source & 0x00FF000000000000) >> 40) | + ((source & 0xFF00000000000000) >> 56); + } + } +} diff --git a/GVFS/FastFetch/Git/GitIndexGenerator.cs b/GVFS/FastFetch/Git/GitIndexGenerator.cs index 0ed30371ed..18c2a0d496 100644 --- a/GVFS/FastFetch/Git/GitIndexGenerator.cs +++ b/GVFS/FastFetch/Git/GitIndexGenerator.cs @@ -1,10 +1,10 @@ using GVFS.Common; using GVFS.Common.Git; -using GVFS.Common.Physical.Git; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; @@ -14,14 +14,15 @@ namespace FastFetch.Git public class GitIndexGenerator { private const long EntryCountOffset = 8; - + + private const ushort ExtendedBit = 0x4000; + private const ushort SkipWorktreeBit = 0x4000; + private static readonly byte[] PaddingBytes = new byte[8]; private static readonly byte[] IndexHeader = new byte[] { (byte)'D', (byte)'I', (byte)'R', (byte)'C', // Magic Signature - 0, 0, 0, 2, // Version - 0, 0, 0, 0 // Number of Entries }; // We can't accurated fill times and length in realtime, so we block write the zeroes and probably save time. @@ -60,11 +61,11 @@ public GitIndexGenerator(ITracer tracer, Enlistment enlistment, bool shouldHashI public bool HasFailures { get; private set; } - public void CreateFromHeadTree() + public void CreateFromHeadTree(uint indexVersion, HashSet sparseCheckoutEntries = null) { using (ITracer updateIndexActivity = this.tracer.StartActivity("CreateFromHeadTree", EventLevel.Informational)) { - Thread entryWritingThread = new Thread(this.WriteAllEntries); + Thread entryWritingThread = new Thread(() => this.WriteAllEntries(indexVersion, sparseCheckoutEntries)); entryWritingThread.Start(); GitProcess git = new GitProcess(this.enlistment); @@ -94,7 +95,7 @@ private void EnqueueEntriesFromLsTree(string line) } } - private void WriteAllEntries() + private void WriteAllEntries(uint version, HashSet sparseCheckoutEntries) { try { @@ -102,11 +103,18 @@ private void WriteAllEntries() using (BinaryWriter writer = new BinaryWriter(indexStream)) { writer.Write(IndexHeader); + writer.Write(EndianHelper.Swap(version)); + writer.Write((uint)0); // Number of entries placeholder + uint lastStringLength = 0; LsTreeEntry entry; while (this.entryQueue.TryTake(out entry, millisecondsTimeout: -1)) { - this.WriteEntry(writer, entry.Sha, entry.Filename); + bool skipWorkTree = + sparseCheckoutEntries != null && + !sparseCheckoutEntries.Contains(entry.Filename) && + !sparseCheckoutEntries.Contains(this.GetDirectoryNameForGitPath(entry.Filename)); + this.WriteEntry(writer, version, entry.Sha, entry.Filename, skipWorkTree, ref lastStringLength); } // Update entry count @@ -125,32 +133,91 @@ private void WriteAllEntries() } } - private void WriteEntry(BinaryWriter writer, string sha, string filename) + private string GetDirectoryNameForGitPath(string filename) + { + int idx = filename.LastIndexOf('/'); + if (idx < 0) + { + return "/"; + } + + return filename.Substring(0, idx + 1); + } + + private void WriteEntry(BinaryWriter writer, uint version, string sha, string filename, bool skipWorktree, ref uint lastStringLength) { + long startPosition = writer.BaseStream.Position; + this.entryCount++; writer.Write(EntryHeader, 0, EntryHeader.Length); - + writer.Write(SHA1Util.BytesFromHexString(sha)); byte[] filenameBytes = Encoding.UTF8.GetBytes(filename); - writer.Write(EndianHelper.Swap((ushort)(filenameBytes.Length & 0xFFF))); + + ushort flags = (ushort)(filenameBytes.Length & 0xFFF); + flags |= version >= 3 && skipWorktree ? ExtendedBit : (ushort)0; + writer.Write(EndianHelper.Swap(flags)); + + if (version >= 3 && skipWorktree) + { + writer.Write(EndianHelper.Swap(SkipWorktreeBit)); + } + + if (version >= 4) + { + this.WriteReplaceLength(writer, lastStringLength); + lastStringLength = (uint)filenameBytes.Length; + } writer.Write(filenameBytes); - - const long EntryLengthWithoutFilename = 62; - // Between 1 and 8 padding bytes. - int numPaddingBytes = 8 - ((int)(EntryLengthWithoutFilename + filenameBytes.Length) % 8); - if (numPaddingBytes == 0) + writer.Flush(); + long endPosition = writer.BaseStream.Position; + + // Version 4 requires a nul-terminated string. + int numPaddingBytes = 1; + if (version < 4) { - numPaddingBytes = 8; + // Version 2-3 has between 1 and 8 padding bytes including nul-terminator. + numPaddingBytes = 8 - ((int)(endPosition - startPosition) % 8); + if (numPaddingBytes == 0) + { + numPaddingBytes = 8; + } } writer.Write(PaddingBytes, 0, numPaddingBytes); + writer.Flush(); } + private void WriteReplaceLength(BinaryWriter writer, uint value) + { + List bytes = new List(); + do + { + byte nextByte = (byte)(value & 0x7F); + value = value >> 7; + bytes.Add(nextByte); + } + while (value != 0); + + bytes.Reverse(); + for (int i = 0; i < bytes.Count; ++i) + { + byte toWrite = bytes[i]; + if (i < bytes.Count - 1) + { + toWrite -= 1; + toWrite |= 0x80; + } + + writer.Write(toWrite); + } + } + private void AppendIndexSha() { byte[] sha = this.GetIndexHash(); @@ -186,6 +253,11 @@ private void ReplaceExistingIndex() private class LsTreeEntry { + public LsTreeEntry() + { + this.Filename = string.Empty; + } + public string Filename { get; private set; } public string Sha { get; private set; } diff --git a/GVFS/FastFetch/Git/GitPackIndex.cs b/GVFS/FastFetch/Git/GitPackIndex.cs index 835bf26ca9..b9486884d9 100644 --- a/GVFS/FastFetch/Git/GitPackIndex.cs +++ b/GVFS/FastFetch/Git/GitPackIndex.cs @@ -1,5 +1,4 @@ -using GVFS.Common.Physical.Git; -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; diff --git a/GVFS/FastFetch/Git/RefSpecHelpers.cs b/GVFS/FastFetch/Git/RefSpecHelpers.cs deleted file mode 100644 index 186d7366ff..0000000000 --- a/GVFS/FastFetch/Git/RefSpecHelpers.cs +++ /dev/null @@ -1,41 +0,0 @@ -using GVFS.Common; -using GVFS.Common.Git; -using GVFS.Common.Tracing; -using Microsoft.Diagnostics.Tracing; -using System; -using System.Linq; - -namespace FastFetch.Git -{ - public static class RefSpecHelpers - { - public const string RefsHeadsGitPath = "refs/heads/"; - - public static bool UpdateRefSpec(ITracer tracer, Enlistment enlistment, string branchOrCommit, GitRefs refs) - { - using (ITracer activity = tracer.StartActivity("UpdateRefSpec", EventLevel.Informational, Keywords.Telemetry, metadata: null)) - { - const string OriginRefMapSettingName = "remote.origin.fetch"; - - // We must update the refspec to get proper "git pull" functionality. - string localBranch = branchOrCommit.StartsWith(RefsHeadsGitPath) ? branchOrCommit : (RefsHeadsGitPath + branchOrCommit); - string remoteBranch = refs.GetBranchRefPairs().Single().Key; - string refSpec = "+" + localBranch + ":" + remoteBranch; - - GitProcess git = new GitProcess(enlistment); - - // Replace all ref-specs this - // * ensures the default refspec (remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*) is removed which avoids some "git fetch/pull" failures - // * gives added "git fetch" performance since git will only fetch the branch provided in the refspec. - GitProcess.Result setResult = git.SetInLocalConfig(OriginRefMapSettingName, refSpec, replaceAll: true); - if (setResult.HasErrors) - { - activity.RelatedError("Could not update ref spec to {0}: {1}", refSpec, setResult.Errors); - return false; - } - } - - return true; - } - } -} diff --git a/GVFS/FastFetch/Git/UpdateRefsHelper.cs b/GVFS/FastFetch/Git/UpdateRefsHelper.cs deleted file mode 100644 index aa3c3d1fd2..0000000000 --- a/GVFS/FastFetch/Git/UpdateRefsHelper.cs +++ /dev/null @@ -1,55 +0,0 @@ -using GVFS.Common; -using GVFS.Common.Git; -using GVFS.Common.Tracing; -using Microsoft.Diagnostics.Tracing; -using System; - -namespace FastFetch.Jobs -{ - public class UpdateRefsHelper - { - private const string AreaPath = nameof(UpdateRefsHelper); - - private Enlistment enlistment; - - public UpdateRefsHelper(Enlistment enlistment) - { - this.enlistment = enlistment; - } - - /// True on success, false otherwise - public bool UpdateRef(ITracer tracer, string refName, string targetCommitish) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("RefName", refName); - metadata.Add("TargetCommitish", targetCommitish); - using (ITracer activity = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata)) - { - GitProcess gitProcess = new GitProcess(this.enlistment); - GitProcess.Result result = null; - if (this.IsSymbolicRef(targetCommitish)) - { - // Using update-ref with a branch name will leave a SHA in the ref file which detaches HEAD, so use symbolic-ref instead. - result = gitProcess.UpdateBranchSymbolicRef(refName, targetCommitish); - } - else - { - result = gitProcess.UpdateBranchSha(refName, targetCommitish); - } - - if (result.HasErrors) - { - activity.RelatedError(result.Errors); - return false; - } - - return true; - } - } - - private bool IsSymbolicRef(string targetCommitish) - { - return targetCommitish.StartsWith("refs/", StringComparison.OrdinalIgnoreCase); - } - } -} diff --git a/GVFS/FastFetch/GitEnlistment.cs b/GVFS/FastFetch/GitEnlistment.cs index e4c32a2249..fcacbb3857 100644 --- a/GVFS/FastFetch/GitEnlistment.cs +++ b/GVFS/FastFetch/GitEnlistment.cs @@ -1,19 +1,17 @@ using GVFS.Common; using System; using System.IO; -using System.Linq; namespace FastFetch { public class GitEnlistment : Enlistment { - private GitEnlistment(string repoRoot, string cacheBaseUrl, string gitBinPath) + private GitEnlistment(string repoRoot, string gitBinPath) : base( repoRoot, repoRoot, Path.Combine(repoRoot, GVFSConstants.DotGit.Objects.Root), null, - cacheBaseUrl, gitBinPath, gvfsHooksRoot: null) { @@ -24,19 +22,12 @@ public string FastFetchLogRoot get { return Path.Combine(this.EnlistmentRoot, GVFSConstants.DotGit.Root, ".fastfetch"); } } - public static GitEnlistment CreateFromCurrentDirectory(string objectsEndpoint, string gitBinPath) + public static GitEnlistment CreateFromCurrentDirectory(string gitBinPath) { - DirectoryInfo dirInfo = new DirectoryInfo(Environment.CurrentDirectory); - while (dirInfo != null && dirInfo.Exists) + string root = Paths.GetGitEnlistmentRoot(Environment.CurrentDirectory); + if (root != null) { - DirectoryInfo[] dotGitDirs = dirInfo.GetDirectories(GVFSConstants.DotGit.Root); - - if (dotGitDirs.Count() == 1) - { - return new GitEnlistment(dirInfo.FullName, objectsEndpoint, gitBinPath); - } - - dirInfo = dirInfo.Parent; + return new GitEnlistment(root, gitBinPath); } return null; diff --git a/GVFS/FastFetch/Index.cs b/GVFS/FastFetch/Index.cs index ed895aca51..1273e6ae7e 100644 --- a/GVFS/FastFetch/Index.cs +++ b/GVFS/FastFetch/Index.cs @@ -7,8 +7,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using FastFetch.Git; using GVFS.Common; -using GVFS.Common.Physical.Git; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; @@ -31,63 +31,52 @@ public class Index // Index default names private const string UpdatedIndexName = "index.updated"; - private const string BackupIndexName = "index.backup"; + + private static readonly byte[] MagicSignature = new byte[] { (byte)'D', (byte)'I', (byte)'R', (byte)'C' }; // Location of the version marker file private readonly string versionMarkerFile; + + private readonly bool readOnly; // Index paths - private string indexPath; - private string updatedIndexPath; - private string backupIndexPath; + private readonly string indexPath; + private readonly string updatedIndexPath; + + private readonly ITracer tracer; + private readonly string repoRoot; - private bool indexReadOnly; - private ITracer tracer; private Dictionary indexEntryOffsets; - private string repoRoot; - private MemoryMappedFile indexMapping; - private MemoryMappedViewAccessor indexView; - private uint indexVersion; private uint entryCount; /// - /// Creates a new Index object to parse the current index - /// Note that this constructor has a pretty specific use case. When this constructor is used, - /// the current index is about to be replaced (by the caller) with a new index. So the current - /// .git\index will be moved (by this constructor) to a backup location so it can be used to - /// populate the new index. + /// Creates a new Index object to parse the specified index file /// - /// - /// public Index( - string repoRoot, - ITracer tracer) - : this(repoRoot, tracer, indexReadOnly: false, indexFullPath: null, backupIndexFullPath: null, backupIndex: true) - { - this.MoveIndexToBackup(); - } - - private Index( string repoRoot, ITracer tracer, - bool indexReadOnly, string indexFullPath, - string backupIndexFullPath, - bool backupIndex) + bool readOnly) { this.tracer = tracer; this.repoRoot = repoRoot; - this.indexReadOnly = indexReadOnly; - this.indexPath = indexFullPath ?? Path.Combine(repoRoot, GVFSConstants.DotGit.Index); - this.updatedIndexPath = indexReadOnly ? this.indexPath : Path.Combine(repoRoot, GVFSConstants.DotGit.Root, UpdatedIndexName); - if (backupIndex && !indexReadOnly) + this.indexPath = indexFullPath; + this.readOnly = readOnly; + + if (this.readOnly) + { + this.updatedIndexPath = this.indexPath; + } + else { - this.backupIndexPath = backupIndexFullPath ?? Path.Combine(repoRoot, GVFSConstants.DotGit.Root, BackupIndexName); + this.updatedIndexPath = Path.Combine(repoRoot, GVFSConstants.DotGit.Root, UpdatedIndexName); } this.versionMarkerFile = Path.Combine(this.repoRoot, GVFSConstants.DotGit.Root, ".fastfetch", "VersionMarker"); } + public uint IndexVersion { get; private set; } + /// /// Updates entries in the current index with file sizes and times /// Algorithm: @@ -103,73 +92,81 @@ public class Index /// /// A collection of added or edited files /// Set to true if the working tree is known good and can be used during the update. - public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLocalFiles, bool allowUpdateFromWorkingTree) + /// An optional index to source entry values from + public void UpdateFileSizesAndTimes(BlockingCollection addedOrEditedLocalFiles, bool allowUpdateFromWorkingTree, bool shouldSignIndex, Index backupIndex = null) { + if (this.readOnly) + { + throw new InvalidOperationException("Cannot update a readonly index."); + } + using (ITracer activity = this.tracer.StartActivity("UpdateFileSizesAndTimes", EventLevel.Informational, Keywords.Telemetry, null)) { - this.CreateWorkingFiles(); + File.Copy(this.indexPath, this.updatedIndexPath, overwrite: true); this.Parse(); - bool previousIndexFound = false; bool anyEntriesUpdated = false; - - if (this.IsFastFetchVersionMarkerCurrent()) + + using (MemoryMappedFile mmf = this.GetMemoryMappedFile()) + using (MemoryMappedViewAccessor indexView = mmf.CreateViewAccessor()) { // Only populate from the previous index if we believe it's good to populate from // For now, a current FastFetch version marker is the only criteria - anyEntriesUpdated |= this.UpdateFileInformationFromBackup(allowUpdateFromWorkingTree, out previousIndexFound); - if (previousIndexFound && (addedOrEditedLocalFiles != null)) + if (backupIndex != null) { - // always update these files from disk or the index won't have good information - // for them and they'll show as modified even those not actually modified. - anyEntriesUpdated |= this.UpdateFileInformationFromDiskForFiles(addedOrEditedLocalFiles); + if (this.IsFastFetchVersionMarkerCurrent()) + { + using (this.tracer.StartActivity("UpdateFileInformationFromPreviousIndex", EventLevel.Informational, Keywords.Telemetry, null)) + { + anyEntriesUpdated |= this.UpdateFileInformationForAllEntries(indexView, backupIndex, allowUpdateFromWorkingTree); + } + + if (addedOrEditedLocalFiles != null) + { + // always update these files from disk or the index won't have good information + // for them and they'll show as modified even those not actually modified. + anyEntriesUpdated |= this.UpdateFileInformationFromDiskForFiles(indexView, addedOrEditedLocalFiles); + } + } + } + else if (allowUpdateFromWorkingTree) + { + // If we didn't update from a previous index, update from the working tree if allowed. + anyEntriesUpdated |= this.UpdateFileInformationFromWorkingTree(indexView); } - } - // If we didn't update from a previous index, update from the working tree if allowed. - if (!previousIndexFound && allowUpdateFromWorkingTree) - { - anyEntriesUpdated |= this.UpdateFileInformationFromWorkingTree(); + indexView.Flush(); } if (anyEntriesUpdated) { - this.MoveUpdatedIndexToFinalLocation(); + this.MoveUpdatedIndexToFinalLocation(shouldSignIndex); + } + else + { + File.Delete(this.updatedIndexPath); } } } - - private void MoveIndexToBackup() + + public void Parse() { - if (this.backupIndexPath != null) + using (ITracer activity = this.tracer.StartActivity("ParseIndex", EventLevel.Informational, Keywords.Telemetry, new EventMetadata() { { "Index", this.updatedIndexPath } })) { - if (File.Exists(this.indexPath)) + using (Stream indexStream = new FileStream(this.updatedIndexPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - // Note that this moves the current index, leaving nothing behind - // This is intentional as we only need it for the purpose of updating the - // new index and leaving it behind can make updating slower. - this.tracer.RelatedEvent(EventLevel.Informational, "CreateBackup", new EventMetadata() { { "BackupIndexName", this.backupIndexPath } }); - File.Delete(this.backupIndexPath); - File.Move(this.indexPath, this.backupIndexPath); - } - else - { - this.tracer.RelatedEvent(EventLevel.Informational, "CreateBackup", new EventMetadata() { { "BackupIndexName", "none" } }); - this.backupIndexPath = null; + this.ParseIndex(indexStream); } } } - private void CreateWorkingFiles() + private MemoryMappedFile GetMemoryMappedFile() { - if (!this.indexReadOnly) - { - File.Copy(this.indexPath, this.updatedIndexPath, overwrite: true); - } + return MemoryMappedFile.CreateFromFile(this.updatedIndexPath, FileMode.Open); } - private bool UpdateFileInformationFromWorkingTree() + private bool UpdateFileInformationFromWorkingTree(MemoryMappedViewAccessor indexView) { long updatedEntries = 0; @@ -185,7 +182,7 @@ private bool UpdateFileInformationFromWorkingTree() long offset; if (this.indexEntryOffsets.TryGetValue(gitPath, out offset)) { - IndexEntry indexEntry = new IndexEntry(this, offset); + IndexEntry indexEntry = new IndexEntry(indexView, offset); indexEntry.Mtime = file.LastWriteTimeUtc; indexEntry.Ctime = file.CreationTimeUtc; indexEntry.Size = (uint)file.Length; @@ -193,109 +190,106 @@ private bool UpdateFileInformationFromWorkingTree() } } }); - - this.indexView.Flush(); } return updatedEntries > 0; } - private bool UpdateFileInformationFromDiskForFiles(BlockingCollection addedOrEditedLocalFiles) + private bool UpdateFileInformationFromDiskForFiles(MemoryMappedViewAccessor indexView, BlockingCollection addedOrEditedLocalFiles) { long updatedEntriesFromDisk = 0; using (ITracer activity = this.tracer.StartActivity("UpdateDownloadedFiles", EventLevel.Informational, Keywords.Telemetry, null)) { Parallel.ForEach( - addedOrEditedLocalFiles, - (localPath) => - { - string gitPath = localPath.FromWindowsFullPathToGitRelativePath(this.repoRoot); - long offset; - if (this.indexEntryOffsets.TryGetValue(gitPath, out offset)) + addedOrEditedLocalFiles, + (localPath) => { - UpdateEntryFromDisk(this, localPath, offset, ref updatedEntriesFromDisk); - } - }); + string gitPath = localPath.FromWindowsFullPathToGitRelativePath(this.repoRoot); + long offset; + if (this.indexEntryOffsets.TryGetValue(gitPath, out offset)) + { + if (TryUpdateEntryFromDisk(indexView, localPath, offset)) + { + Interlocked.Increment(ref updatedEntriesFromDisk); + } + } + }); } - this.tracer.RelatedEvent(EventLevel.Informational, "UpdateFileInformationFromDiskForFiles", new EventMetadata() { { "UpdatedFromDisk", updatedEntriesFromDisk } }, Keywords.Telemetry); + this.tracer.RelatedEvent(EventLevel.Informational, "UpdateIndexFileInformation", new EventMetadata() { { "UpdatedFromDisk", updatedEntriesFromDisk } }, Keywords.Telemetry); return updatedEntriesFromDisk > 0; } - - private bool UpdateFileInformationFromBackup(bool shouldAlsoTryPopulateFromDisk, out bool indexFound) - { - indexFound = (this.backupIndexPath != null) && File.Exists(this.backupIndexPath); - if (!indexFound) - { - return false; - } - - using (ITracer activity = this.tracer.StartActivity("UpdateFileInformationFromPreviousIndex", EventLevel.Informational, Keywords.Telemetry, null)) - { - Index backupIndex = new Index(this.repoRoot, this.tracer, indexReadOnly: true, indexFullPath: this.backupIndexPath, backupIndexFullPath: null, backupIndex: false); - backupIndex.Parse(); - return this.UpdateFileInformationFromAnotherIndex(backupIndex, shouldAlsoTryPopulateFromDisk); - } - } - - private bool UpdateFileInformationFromAnotherIndex(Index otherIndex, bool shouldAlsoTryPopulateFromDisk) + + private bool UpdateFileInformationForAllEntries(MemoryMappedViewAccessor indexView, Index otherIndex, bool shouldAlsoTryPopulateFromDisk) { long updatedEntriesFromOtherIndex = 0; - foreach (KeyValuePair i in this.indexEntryOffsets) + long updatedEntriesFromDisk = 0; + + using (MemoryMappedFile mmf = otherIndex.GetMemoryMappedFile()) + using (MemoryMappedViewAccessor otherIndexView = mmf.CreateViewAccessor()) { - string currentIndexFilename = i.Key; - long currentIndexOffset = i.Value; - if (IndexEntry.HasUninitializedCTimeEntry(this, currentIndexOffset)) - { - long otherIndexOffset; - if (otherIndex.indexEntryOffsets.TryGetValue(currentIndexFilename, out otherIndexOffset)) + Parallel.ForEach( + this.indexEntryOffsets, + entry => { - if (!IndexEntry.HasUninitializedCTimeEntry(otherIndex, otherIndexOffset)) + string currentIndexFilename = entry.Key; + long currentIndexOffset = entry.Value; + if (!IndexEntry.HasInitializedCTimeEntry(indexView, currentIndexOffset)) { - IndexEntry currentIndexEntry = new IndexEntry(this, currentIndexOffset); - IndexEntry otherIndexEntry = new IndexEntry(otherIndex, otherIndexOffset); - currentIndexEntry.CtimeSeconds = otherIndexEntry.CtimeSeconds; - currentIndexEntry.CtimeNanosecondFraction = otherIndexEntry.CtimeNanosecondFraction; - currentIndexEntry.MtimeSeconds = otherIndexEntry.MtimeSeconds; - currentIndexEntry.MtimeNanosecondFraction = otherIndexEntry.MtimeNanosecondFraction; - currentIndexEntry.Size = otherIndexEntry.Size; - ++updatedEntriesFromOtherIndex; + long otherIndexOffset; + if (otherIndex.indexEntryOffsets.TryGetValue(currentIndexFilename, out otherIndexOffset)) + { + if (IndexEntry.HasInitializedCTimeEntry(otherIndexView, otherIndexOffset)) + { + IndexEntry currentIndexEntry = new IndexEntry(indexView, currentIndexOffset); + IndexEntry otherIndexEntry = new IndexEntry(otherIndexView, otherIndexOffset); + + currentIndexEntry.CtimeSeconds = otherIndexEntry.CtimeSeconds; + currentIndexEntry.CtimeNanosecondFraction = otherIndexEntry.CtimeNanosecondFraction; + currentIndexEntry.MtimeSeconds = otherIndexEntry.MtimeSeconds; + currentIndexEntry.MtimeNanosecondFraction = otherIndexEntry.MtimeNanosecondFraction; + currentIndexEntry.Size = otherIndexEntry.Size; + + Interlocked.Increment(ref updatedEntriesFromOtherIndex); + } + } + else if (shouldAlsoTryPopulateFromDisk) + { + string localPath = currentIndexFilename.FromGitRelativePathToWindowsFullPath(this.repoRoot); + if (TryUpdateEntryFromDisk(indexView, localPath, entry.Value)) + { + Interlocked.Increment(ref updatedEntriesFromDisk); + } + } } - } - } + }); } - long updatedEntriesFromDisk = 0; - if (shouldAlsoTryPopulateFromDisk) - { - Parallel.ForEach( - this.indexEntryOffsets.Where(entry => IndexEntry.HasUninitializedCTimeEntry(this, entry.Value)), - (entry) => + this.tracer.RelatedEvent( + EventLevel.Informational, + "UpdateIndexFileInformation", + new EventMetadata() { - string localPath = entry.Key.FromGitRelativePathToWindowsFullPath(this.repoRoot); - UpdateEntryFromDisk(this, localPath, entry.Value, ref updatedEntriesFromDisk); - }); - - this.tracer.RelatedEvent(EventLevel.Informational, "UpdateFileInformationFromAnotherIndex", new EventMetadata() { { "UpdatedFromOtherIndex", updatedEntriesFromOtherIndex }, { "UpdatedFromDisk", updatedEntriesFromDisk } }, Keywords.Telemetry); - } - - this.indexView.Flush(); + { "UpdatedFromOtherIndex", updatedEntriesFromOtherIndex }, + { "UpdatedFromDisk", updatedEntriesFromDisk } + }, + Keywords.Telemetry); return (updatedEntriesFromOtherIndex > 0) || (updatedEntriesFromDisk > 0); } - private void UpdateEntryFromDisk(Index index, string localPath, long offset, ref long counter) + private bool TryUpdateEntryFromDisk(MemoryMappedViewAccessor indexView, string localPath, long offset) { try { FileInfo file = new FileInfo(localPath); if (file.Exists) { - IndexEntry indexEntry = new IndexEntry(index, offset); + IndexEntry indexEntry = new IndexEntry(indexView, offset); indexEntry.Mtime = file.LastWriteTimeUtc; indexEntry.Ctime = file.CreationTimeUtc; indexEntry.Size = (uint)file.Length; - Interlocked.Increment(ref counter); + return true; } } catch (System.Security.SecurityException) @@ -306,21 +300,31 @@ private void UpdateEntryFromDisk(Index index, string localPath, long offset, ref { // Skip these. } + + return false; } - private void MoveUpdatedIndexToFinalLocation() + private void MoveUpdatedIndexToFinalLocation(bool shouldSignIndex) { - if (this.indexView != null) + if (shouldSignIndex) { - this.indexView.Flush(); - this.indexView.Dispose(); - this.indexView = null; - } + using (ITracer activity = this.tracer.StartActivity("SignIndex", EventLevel.Informational, Keywords.Telemetry, metadata: null)) + { + using (FileStream fs = File.Open(this.updatedIndexPath, FileMode.Open, FileAccess.ReadWrite)) + { + // Truncate the old hash off. The Index class is expected to preserve any existing hash. + fs.SetLength(fs.Length - 20); + using (HashingStream hashStream = new HashingStream(fs)) + { + fs.Position = 0; + hashStream.CopyTo(Stream.Null); + byte[] hash = hashStream.Hash; - if (this.indexMapping != null) - { - this.indexMapping.Dispose(); - this.indexView = null; + // The fs pointer is now where the old hash used to be. Perfect. :) + fs.Write(hash, 0, hash.Length); + } + } + } } this.tracer.RelatedEvent(EventLevel.Informational, "MoveUpdatedIndexToFinalLocation", new EventMetadata() { { "UpdatedIndex", this.updatedIndexPath }, { "Index", this.indexPath } }); @@ -337,12 +341,13 @@ private void WriteFastFetchIndexVersionMarker() File.SetAttributes(this.versionMarkerFile, FileAttributes.Normal); } + Directory.CreateDirectory(Path.GetDirectoryName(this.versionMarkerFile)); File.WriteAllText(this.versionMarkerFile, CurrentFastFetchIndexVersion.ToString(), Encoding.ASCII); File.SetAttributes(this.versionMarkerFile, FileAttributes.ReadOnly); this.tracer.RelatedEvent(EventLevel.Informational, "MarkerWritten", new EventMetadata() { { "Version", CurrentFastFetchIndexVersion } }); } - private bool IsFastFetchVersionMarkerCurrent() + private bool IsFastFetchVersionMarkerCurrent() { if (File.Exists(this.versionMarkerFile)) { @@ -357,30 +362,28 @@ private bool IsFastFetchVersionMarkerCurrent() return false; } - private void Parse() - { - using (ITracer activity = this.tracer.StartActivity("ParseIndex", EventLevel.Informational, Keywords.Telemetry, new EventMetadata() { { "Index", this.updatedIndexPath } })) - { - this.indexMapping = MemoryMappedFile.CreateFromFile(this.updatedIndexPath, FileMode.Open); - this.indexView = this.indexMapping.CreateViewAccessor(); - using (MemoryMappedViewStream indexStream = this.indexMapping.CreateViewStream()) - { - this.ParseIndex(indexStream, updateOffsetsOnly: false); - } - } - } - - private void ParseIndex(MemoryMappedViewStream indexStream, bool updateOffsetsOnly = false) + private void ParseIndex(Stream indexStream) { byte[] buffer = new byte[40]; indexStream.Position = 0; byte[] signature = new byte[4]; indexStream.Read(signature, 0, 4); - this.indexVersion = this.ReadUInt32(buffer, indexStream); + if (!Enumerable.SequenceEqual(MagicSignature, signature)) + { + throw new InvalidDataException("Incorrect magic signature for index: " + string.Join(string.Empty, signature.Select(c => (char)c))); + } + + this.IndexVersion = this.ReadUInt32(buffer, indexStream); + + if (this.IndexVersion < 2 || this.IndexVersion > 4) + { + throw new InvalidDataException("Unsupported index version: " + this.IndexVersion); + } + this.entryCount = this.ReadUInt32(buffer, indexStream); - this.tracer.RelatedEvent(EventLevel.Informational, "IndexData", new EventMetadata() { { "Index", this.updatedIndexPath }, { "Version", this.indexVersion }, { "entryCount", this.entryCount } }, Keywords.Telemetry); + this.tracer.RelatedEvent(EventLevel.Informational, "IndexData", new EventMetadata() { { "Index", this.updatedIndexPath }, { "Version", this.IndexVersion }, { "entryCount", this.entryCount } }, Keywords.Telemetry); this.indexEntryOffsets = new Dictionary((int)this.entryCount, StringComparer.OrdinalIgnoreCase); @@ -400,18 +403,18 @@ private void ParseIndex(MemoryMappedViewStream indexStream, bool updateOffsetsOn ushort flags = this.ReadUInt16(buffer, indexStream); bool isExtended = (flags & ExtendedBit) == ExtendedBit; - int pathLength = (ushort)(((flags << 20) >> 20) & 4095); + ushort pathLength = (ushort)(flags & 0xFFF); entryLength += pathLength; bool skipWorktree = false; - if (isExtended && (this.indexVersion > 2)) + if (isExtended && (this.IndexVersion > 2)) { ushort extendedFlags = this.ReadUInt16(buffer, indexStream); skipWorktree = (extendedFlags & SkipWorktreeBit) == SkipWorktreeBit; entryLength += 2; } - if (this.indexVersion == 4) + if (this.IndexVersion == 4) { int replaceLength = this.ReadReplaceLength(indexStream); int replaceIndex = previousPathLength - replaceLength; @@ -425,7 +428,7 @@ private void ParseIndex(MemoryMappedViewStream indexStream, bool updateOffsetsOn indexStream.Read(pathBuffer, 0, pathLength + numNulBytes); } - if (!skipWorktree) + if (!skipWorktree) { // Examine only the things we're not skipping... // Potential Future Perf Optimization: Perform this work on multiple threads. If we take the first byte and % by number of threads, @@ -451,6 +454,10 @@ private int ReadReplaceLength(Stream stream) for (int i = 0; (headerByte & 0x80) != 0; i++) { headerByte = stream.ReadByte(); + if (headerByte < 0) + { + throw new EndOfStreamException("Index file has been truncated."); + } offset += 1; offset = (offset << 7) + (headerByte & 0x7f); @@ -486,11 +493,11 @@ private class IndexEntry private const long UnixEpochMilliseconds = 116444736000000000; private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - private Index index; + private MemoryMappedViewAccessor indexView; - public IndexEntry(Index index, long offset) + public IndexEntry(MemoryMappedViewAccessor indexView, long offset) { - this.index = index; + this.indexView = indexView; this.Offset = offset; } @@ -625,46 +632,30 @@ public bool IsExtended return (this.Flags & Index.ExtendedBit) == Index.ExtendedBit; } } - - public ushort ExtendedFlags - { - get - { - return (this.IsExtended && (this.index.indexVersion > 2)) ? this.ReadUInt16(EntryOffsets.extendedFlags) : (ushort)0; - } - - set - { - if (this.IsExtended) - { - this.WriteUInt32(EntryOffsets.extendedFlags, value); - } - } - } - - public static bool HasUninitializedCTimeEntry(Index index, long offset) + + public static bool HasInitializedCTimeEntry(MemoryMappedViewAccessor indexView, long offset) { - return EndianHelper.Swap(index.indexView.ReadUInt32(offset + (long)EntryOffsets.ctimeSeconds)) == 0; + return EndianHelper.Swap(indexView.ReadUInt32(offset + (long)EntryOffsets.ctimeSeconds)) != 0; } private uint ReadUInt32(EntryOffsets fromOffset) { - return EndianHelper.Swap(this.index.indexView.ReadUInt32(this.Offset + (long)fromOffset)); + return EndianHelper.Swap(this.indexView.ReadUInt32(this.Offset + (long)fromOffset)); } private void WriteUInt32(EntryOffsets fromOffset, uint data) { - this.index.indexView.Write(this.Offset + (long)fromOffset, EndianHelper.Swap(data)); + this.indexView.Write(this.Offset + (long)fromOffset, EndianHelper.Swap(data)); } private ushort ReadUInt16(EntryOffsets fromOffset) { - return EndianHelper.Swap(this.index.indexView.ReadUInt16(this.Offset + (long)fromOffset)); + return EndianHelper.Swap(this.indexView.ReadUInt16(this.Offset + (long)fromOffset)); } private void WriteUInt16(EntryOffsets fromOffset, ushort data) { - this.index.indexView.Write(this.Offset + (long)fromOffset, EndianHelper.Swap(data)); + this.indexView.Write(this.Offset + (long)fromOffset, EndianHelper.Swap(data)); } private IndexEntryTime ToUnixTime(DateTime datetime) @@ -686,7 +677,7 @@ private IndexEntryTime ToUnixTime(DateTime datetime) private DateTime ToWindowsTime(uint seconds, uint nanosecondFraction) { - DateTime time = UnixEpoch.AddSeconds(seconds).AddMilliseconds(nanosecondFraction * 1000000); + DateTime time = UnixEpoch.AddSeconds(seconds).AddMilliseconds(nanosecondFraction / 1000000); return time; } @@ -697,4 +688,4 @@ private class IndexEntryTime } } } -} +} \ No newline at end of file diff --git a/GVFS/FastFetch/Jobs/BatchObjectDownloadJob.cs b/GVFS/FastFetch/Jobs/BatchObjectDownloadJob.cs index af78318b6a..fb65823710 100644 --- a/GVFS/FastFetch/Jobs/BatchObjectDownloadJob.cs +++ b/GVFS/FastFetch/Jobs/BatchObjectDownloadJob.cs @@ -2,6 +2,7 @@ using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.Http; +using GVFS.Common.NetworkStreams; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; @@ -148,7 +149,7 @@ protected override void DoAfterWork() this.AvailablePacks.Add(new IndexPackRequest(fileName, request)); break; case GitObjectContentType.BatchedLooseObjects: - OnLooseObject onLooseObject = (objectStream, sha1) => + BatchedLooseObjectDeserializer.OnLooseObject onLooseObject = (objectStream, sha1) => { this.gitObjects.WriteLooseObject( objectStream, diff --git a/GVFS/FastFetch/Jobs/CheckoutJob.cs b/GVFS/FastFetch/Jobs/CheckoutJob.cs index 9469b835c8..f08358bdec 100644 --- a/GVFS/FastFetch/Jobs/CheckoutJob.cs +++ b/GVFS/FastFetch/Jobs/CheckoutJob.cs @@ -1,7 +1,7 @@ using FastFetch.Git; using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Git; -using GVFS.Common.Physical.FileSystem; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; diff --git a/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs b/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs index 3b597b21b7..0464c9966e 100644 --- a/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs +++ b/GVFS/FastFetch/Jobs/FindMissingBlobsJob.cs @@ -55,15 +55,18 @@ protected override void DoWork() { while (this.inputQueue.TryTake(out blobId, Timeout.Infinite)) { - if (!repo.ObjectExists(blobId)) + if (this.alreadyFoundBlobIds.Add(blobId)) { - Interlocked.Increment(ref this.missingBlobCount); - this.DownloadQueue.Add(blobId); - } - else - { - Interlocked.Increment(ref this.availableBlobCount); - this.AvailableBlobs.Add(blobId); + if (!repo.ObjectExists(blobId)) + { + Interlocked.Increment(ref this.missingBlobCount); + this.DownloadQueue.Add(blobId); + } + else + { + Interlocked.Increment(ref this.availableBlobCount); + this.AvailableBlobs.Add(blobId); + } } } } diff --git a/GVFS/GVFS.Common/AntiVirusExclusions.cs b/GVFS/GVFS.Common/AntiVirusExclusions.cs index 8257b9bccf..83af80a62b 100644 --- a/GVFS/GVFS.Common/AntiVirusExclusions.cs +++ b/GVFS/GVFS.Common/AntiVirusExclusions.cs @@ -72,7 +72,7 @@ private static bool TryGetKnownExclusions(out string[] exclusions, out string er private static ProcessResult CallPowershellCommand(string command) { - return ProcessHelper.Run("powershell", "-NoProfile -Command \"& { " + command + " }\""); + return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); } private static string CleanPath(string path) diff --git a/GVFS/GVFS.Common/ConcurrentHashSet.cs b/GVFS/GVFS.Common/ConcurrentHashSet.cs index fe252db130..fe1b17f719 100644 --- a/GVFS/GVFS.Common/ConcurrentHashSet.cs +++ b/GVFS/GVFS.Common/ConcurrentHashSet.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections; +using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; diff --git a/GVFS/GVFS.Common/ConsoleHelper.cs b/GVFS/GVFS.Common/ConsoleHelper.cs index 16e7037025..e3f01a4406 100644 --- a/GVFS/GVFS.Common/ConsoleHelper.cs +++ b/GVFS/GVFS.Common/ConsoleHelper.cs @@ -35,7 +35,7 @@ private enum FileType : uint string message, TextWriter output, bool showSpinner, - bool suppressGvfsLogMessage = false, + string gvfsLogEnlistmentRoot, int initialDelayMs = 0) { Func actionResultAction = @@ -49,8 +49,8 @@ private enum FileType : uint message, output, showSpinner, - suppressGvfsLogMessage, - initialDelayMs); + gvfsLogEnlistmentRoot, + initialDelayMs: initialDelayMs); return result == ActionResult.Success; } @@ -60,8 +60,8 @@ private enum FileType : uint string message, TextWriter output, bool showSpinner, - bool suppressGvfsLogMessage = false, - int initialDelayMs = 0) + string gvfsLogEnlistmentRoot, + int initialDelayMs) { ActionResult result = ActionResult.Failure; bool initialMessageWritten = false; @@ -148,7 +148,7 @@ private enum FileType : uint output.Write("\r{0}...", message); } - output.WriteLine("Failed" + (suppressGvfsLogMessage ? string.Empty : ". Run 'gvfs log' for more info.")); + output.WriteLine("Failed" + (gvfsLogEnlistmentRoot == null ? string.Empty : ". " + GetGVFSLogMessage(gvfsLogEnlistmentRoot))); break; } } @@ -161,6 +161,11 @@ public static bool IsConsoleOutputRedirectedToFile() return FileType.Disk == GetFileType(GetStdHandle(StdHandle.Stdout)); } + public static string GetGVFSLogMessage(string enlistmentRoot) + { + return "Run 'gvfs log " + enlistmentRoot + "' for more info."; + } + [DllImport("kernel32.dll")] private static extern IntPtr GetStdHandle(StdHandle std); diff --git a/GVFS/GVFS.Common/Enlistment.cs b/GVFS/GVFS.Common/Enlistment.cs index 8425779c8c..5db1e6da77 100644 --- a/GVFS/GVFS.Common/Enlistment.cs +++ b/GVFS/GVFS.Common/Enlistment.cs @@ -8,8 +8,6 @@ namespace GVFS.Common public abstract class Enlistment { private const string DeprecatedObjectsEndpointGitConfigName = "gvfs.objects-endpoint"; - - private const string GVFSGitConfigPrefix = "gvfs."; private const string CacheEndpointGitConfigSuffix = ".cache-server-url"; protected Enlistment( @@ -17,7 +15,6 @@ public abstract class Enlistment string workingDirectoryRoot, string gitObjectsRoot, string repoUrl, - string cacheServerUrl, string gitBinPath, string gvfsHooksRoot) { @@ -43,13 +40,17 @@ public abstract class Enlistment GitProcess.Result originResult = new GitProcess(this).GetOriginUrl(); if (originResult.HasErrors) { + if (originResult.Errors.Length == 0) + { + throw new InvalidRepoException("Could not get origin url. remote 'origin' is not configured for this repo.'"); + } + throw new InvalidRepoException("Could not get origin url. git error: " + originResult.Errors); } - this.RepoUrl = originResult.Output; + this.RepoUrl = originResult.Output.Trim(); } - - this.SetComputedURLs(cacheServerUrl); + this.Authentication = new GitAuthentication(this); } @@ -59,26 +60,11 @@ public abstract class Enlistment public string GitObjectsRoot { get; private set; } public string GitPackRoot { get; private set; } public string RepoUrl { get; } - public string CacheServerUrl { get; private set; } - - public string ObjectsEndpointUrl { get; private set; } - - public string PrefetchEndpointUrl { get; private set; } public string GitBinPath { get; } public string GVFSHooksRoot { get; } public GitAuthentication Authentication { get; } - - public static string StripObjectsEndpointSuffix(string input) - { - if (!string.IsNullOrWhiteSpace(input) && input.EndsWith(GVFSConstants.Endpoints.GVFSObjects)) - { - input = input.Substring(0, input.Length - GVFSConstants.Endpoints.GVFSObjects.Length); - } - - return input; - } public static string GetNewLogFileName(string logsRoot, string prefix) { @@ -101,74 +87,11 @@ public static string GetNewLogFileName(string logsRoot, string prefix) return fullPath; } - - protected static string GetCacheConfigSettingName(string repoUrl) - { - string sectionUrl = - repoUrl.ToLowerInvariant() - .Replace("https://", string.Empty) - .Replace("http://", string.Empty) - .Replace('/', '.'); - - return GVFSGitConfigPrefix + sectionUrl + CacheEndpointGitConfigSuffix; - } - protected string GetCacheServerUrlFromConfig(string repoUrl) - { - GitProcess git = new GitProcess(this); - string cacheConfigName = GetCacheConfigSettingName(repoUrl); - - string cacheServerUrl = this.GetFromConfig(git, cacheConfigName); - if (string.IsNullOrWhiteSpace(cacheServerUrl)) - { - // Try getting from the deprecated setting for compatibility reasons - cacheServerUrl = StripObjectsEndpointSuffix(this.GetFromConfig(git, DeprecatedObjectsEndpointGitConfigName)); - - // Upgrade for future runs, but not at clone time. - if (!string.IsNullOrWhiteSpace(cacheServerUrl) && Directory.Exists(this.WorkingDirectoryRoot)) - { - git.SetInLocalConfig(cacheConfigName, cacheServerUrl); - git.DeleteFromLocalConfig(DeprecatedObjectsEndpointGitConfigName); - } - } - - // Default to uncached url - if (string.IsNullOrWhiteSpace(cacheServerUrl)) - { - return repoUrl; - } - - return cacheServerUrl; - } - - private string GetFromConfig(GitProcess git, string configName) - { - GitProcess.Result result = git.GetFromConfig(configName); - - // Git returns non-zero for non-existent settings and errors. - if (!result.HasErrors) - { - return result.Output.TrimEnd('\n'); - } - else if (result.Errors.Any()) - { - throw new InvalidRepoException("Error while reading '" + configName + "' from config: " + result.Errors); - } - - return null; - } - private void SetComputedPaths() { this.DotGitRoot = Path.Combine(this.WorkingDirectoryRoot, GVFSConstants.DotGit.Root); this.GitPackRoot = Path.Combine(this.GitObjectsRoot, GVFSConstants.DotGit.Objects.Pack.Name); } - - private void SetComputedURLs(string cacheServerUrl) - { - this.CacheServerUrl = !string.IsNullOrWhiteSpace(cacheServerUrl) ? cacheServerUrl : this.GetCacheServerUrlFromConfig(this.RepoUrl); - this.ObjectsEndpointUrl = this.CacheServerUrl + GVFSConstants.Endpoints.GVFSObjects; - this.PrefetchEndpointUrl = this.CacheServerUrl + GVFSConstants.Endpoints.GVFSPrefetch; - } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/EnlistmentUtils.cs b/GVFS/GVFS.Common/EnlistmentUtils.cs deleted file mode 100644 index 75d2f545dc..0000000000 --- a/GVFS/GVFS.Common/EnlistmentUtils.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.IO; -using System.Linq; - -namespace GVFS.Common -{ - public static class EnlistmentUtils - { - public static string GetEnlistmentRoot(string directory) - { - directory = directory.TrimEnd(GVFSConstants.PathSeparator); - DirectoryInfo dirInfo; - - try - { - dirInfo = new DirectoryInfo(directory); - } - catch (Exception) - { - return null; - } - - while (dirInfo != null) - { - if (dirInfo.Exists) - { - DirectoryInfo[] dotGvfsDirs = dirInfo.GetDirectories(GVFSConstants.DotGVFS.Root); - - if (dotGvfsDirs.Count() == 1) - { - return dirInfo.FullName; - } - } - - dirInfo = dirInfo.Parent; - } - - return null; - } - } -} diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index 33e85d38f8..b344c96e02 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -1,4 +1,4 @@ -using GVFS.Common.Physical.FileSystem; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; diff --git a/GVFS/GVFS.Common/Physical/FileSystem/DirectoryItemInfo.cs b/GVFS/GVFS.Common/FileSystem/DirectoryItemInfo.cs similarity index 80% rename from GVFS/GVFS.Common/Physical/FileSystem/DirectoryItemInfo.cs rename to GVFS/GVFS.Common/FileSystem/DirectoryItemInfo.cs index d46db072c3..7fc2326e04 100644 --- a/GVFS/GVFS.Common/Physical/FileSystem/DirectoryItemInfo.cs +++ b/GVFS/GVFS.Common/FileSystem/DirectoryItemInfo.cs @@ -1,4 +1,4 @@ -namespace GVFS.Common.Physical.FileSystem +namespace GVFS.Common.FileSystem { public class DirectoryItemInfo { diff --git a/GVFS/GVFS.Common/Physical/FileSystem/FileProperties.cs b/GVFS/GVFS.Common/FileSystem/FileProperties.cs similarity index 94% rename from GVFS/GVFS.Common/Physical/FileSystem/FileProperties.cs rename to GVFS/GVFS.Common/FileSystem/FileProperties.cs index 3ffd7ff9e3..a6ab702ed1 100644 --- a/GVFS/GVFS.Common/Physical/FileSystem/FileProperties.cs +++ b/GVFS/GVFS.Common/FileSystem/FileProperties.cs @@ -1,7 +1,7 @@ using System; using System.IO; -namespace GVFS.Common.Physical.FileSystem +namespace GVFS.Common.FileSystem { public class FileProperties { diff --git a/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs b/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs new file mode 100644 index 0000000000..8df45d8ae5 --- /dev/null +++ b/GVFS/GVFS.Common/FileSystem/GvFltFilter.cs @@ -0,0 +1,220 @@ +using GVFS.Common.Tracing; +using Microsoft.Win32; +using System; +using System.Runtime.InteropServices; +using System.Security; +using System.ServiceProcess; +using System.Text; + +namespace GVFS.Common.FileSystem +{ + public class GvFltFilter + { + public const RegistryHive GvFltParametersHive = RegistryHive.LocalMachine; + public const string GvFltParametersKey = "SYSTEM\\CurrentControlSet\\Services\\Gvflt\\Parameters"; + public const string GvFltTimeoutValue = "CommandTimeoutInMs"; + private const string EtwArea = nameof(GvFltFilter); + + private const int MinGvFltTimeoutMs = 86400000; + + private const string GvFltName = "gvflt"; + + private const uint OkResult = 0; + private const uint NameCollisionErrorResult = 0x801F0012; + + public static bool TryAttach(ITracer tracer, string root, out string errorMessage) + { + errorMessage = null; + try + { + StringBuilder volumePathName = new StringBuilder(GVFSConstants.MaxPath); + if (!NativeMethods.GetVolumePathName(root, volumePathName, GVFSConstants.MaxPath)) + { + errorMessage = "Could not get volume path name"; + tracer.RelatedError(errorMessage); + return false; + } + + uint result = NativeMethods.FilterAttach(GvFltName, volumePathName.ToString(), null); + if (result != OkResult && result != NameCollisionErrorResult) + { + errorMessage = string.Format("Attaching the filter driver resulted in: {0}", result); + tracer.RelatedError(errorMessage); + return false; + } + } + catch (Exception e) + { + errorMessage = string.Format("Attaching the filter driver resulted in: {0}", e.Message); + tracer.RelatedError(errorMessage); + return false; + } + + return true; + } + + public static bool IsHealthy(out string error, out string warning, ITracer tracer) + { + // TODO 1026787: Record errors\warnings in the event log + + warning = string.Empty; + + if (!IsServiceRunning(out error, tracer)) + { + return false; + } + + CheckTimeoutConfiguration(out warning, tracer); + return true; + } + + public static bool TryGetTimeout(out int timeoutMs, out string error, ITracer tracer = null) + { + timeoutMs = 0; + error = string.Empty; + object value; + try + { + value = ProcessHelper.GetValueFromRegistry(GvFltParametersHive, GvFltParametersKey, GvFltTimeoutValue); + } + catch (UnauthorizedAccessException e) + { + if (tracer != null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", e.ToString()); + metadata.Add("ErrorMessage", "UnauthorizedAccessException while trying to read GvFlt timeout"); + tracer.RelatedError(metadata); + } + + error = "Failed to read GvFlt timeout from registry. " + e.Message; + return false; + } + catch (SecurityException e) + { + if (tracer != null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", e.ToString()); + metadata.Add("ErrorMessage", "SecurityException while trying to read GvFlt timeout"); + tracer.RelatedError(metadata); + } + + error = "Failed to read GvFlt timeout from registry. " + e.Message; + return false; + } + catch (Exception e) + { + if (tracer != null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", e.ToString()); + metadata.Add("ErrorMessage", "Exception while trying to read GvFlt timeout"); + tracer.RelatedError(metadata); + } + + error = "Failed to read GvFlt timeout from registry. " + e.Message; + return false; + } + + try + { + timeoutMs = Convert.ToInt32(value); + } + catch (Exception e) + { + if (tracer != null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", e.ToString()); + metadata.Add("ErrorMessage", "Exception while trying to convert GvFlt timeout to int"); + tracer.RelatedError(metadata); + } + + error = "GvFlt timeout not properly configured, failed to convert value to int: " + e.Message; + return false; + } + + return true; + } + + private static bool IsServiceRunning(out string error, ITracer tracer) + { + error = string.Empty; + + bool gvfltServiceRunning = false; + try + { + ServiceController controller = new ServiceController("gvflt"); + gvfltServiceRunning = controller.Status.Equals(ServiceControllerStatus.Running); + } + catch (InvalidOperationException e) + { + if (tracer != null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("Exception", e.ToString()); + metadata.Add("ErrorMessage", "InvalidOperationException: GvFlt Service was not found"); + tracer.RelatedError(metadata); + } + + error = "Error: GvFlt Service was not found. To resolve, re-install GVFS"; + return false; + } + + if (!gvfltServiceRunning) + { + if (tracer != null) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("ErrorMessage", "GvFlt Service is not running"); + tracer.RelatedError(metadata); + } + + error = "Error: GvFlt Service is not running. To resolve, run \"sc start gvflt\" from an elevated command prompt"; + return false; + } + + return true; + } + + private static void CheckTimeoutConfiguration(out string warning, ITracer tracer) + { + warning = string.Empty; + string error; + int timemoutMs; + if (!TryGetTimeout(out timemoutMs, out error, tracer)) + { + warning = "Warning: Failed to validate GvFlt timeout configuration: " + error; + } + else if (timemoutMs < MinGvFltTimeoutMs) + { + warning = string.Format("Warning: GvFlt timeout not properly configured, timeout {0} less than recommended value {1}", timemoutMs, MinGvFltTimeoutMs); + } + } + + private static class NativeMethods + { + [DllImport("fltlib.dll", CharSet = CharSet.Unicode)] + public static extern uint FilterAttach( + string filterName, + string volumeName, + string instanceName, + uint createdInstanceNameLength = 0, + string createdInstanceName = null); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetVolumePathName( + string volumeName, + StringBuilder volumePathName, + uint bufferLength); + } + } +} diff --git a/GVFS/GVFS.Common/Physical/FileSystem/PhysicalFileSystem.cs b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs similarity index 93% rename from GVFS/GVFS.Common/Physical/FileSystem/PhysicalFileSystem.cs rename to GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs index 82b3301ab8..557443f1fe 100644 --- a/GVFS/GVFS.Common/Physical/FileSystem/PhysicalFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs @@ -1,10 +1,9 @@ using Microsoft.Win32.SafeHandles; -using System; using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; -namespace GVFS.Common.Physical.FileSystem +namespace GVFS.Common.FileSystem { public class PhysicalFileSystem { @@ -83,6 +82,11 @@ public virtual SafeHandle OpenFile(string path, FileMode fileMode, FileAccess fi return NativeMethods.OpenFile(path, fileMode, (NativeMethods.FileAccess)fileAccess, shareMode, (NativeMethods.FileAttributes)attributes); } + public virtual void CreateDirectory(string path) + { + Directory.CreateDirectory(path); + } + public virtual void DeleteDirectory(string path, bool recursive = false) { RecursiveDelete(path); diff --git a/GVFS/GVFS.Common/GVFS.Common.csproj b/GVFS/GVFS.Common/GVFS.Common.csproj index f6f28f8b98..f25911cea3 100644 --- a/GVFS/GVFS.Common/GVFS.Common.csproj +++ b/GVFS/GVFS.Common/GVFS.Common.csproj @@ -69,6 +69,7 @@ + @@ -81,25 +82,28 @@ CommonAssemblyVersion.cs - - + + + - - + + - + + + @@ -109,12 +113,13 @@ - - - + + + + @@ -124,13 +129,11 @@ - - @@ -139,13 +142,11 @@ - - - - - - - + + + + + @@ -159,11 +160,11 @@ - + diff --git a/GVFS/GVFS.Common/GVFSConfig.cs b/GVFS/GVFS.Common/GVFSConfig.cs index 418492d829..7b34c6b7c0 100644 --- a/GVFS/GVFS.Common/GVFSConfig.cs +++ b/GVFS/GVFS.Common/GVFSConfig.cs @@ -1,4 +1,5 @@ -using System; +using GVFS.Common.Http; +using System; using System.Collections.Generic; namespace GVFS.Common @@ -7,6 +8,8 @@ public class GVFSConfig { public IEnumerable AllowedGVFSClientVersions { get; set; } + public IEnumerable CacheServers { get; set; } + public class VersionRange { public Version Min { get; set; } diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index ab3d8d25b7..37d34db59f 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -23,11 +23,22 @@ public static class GVFSConstants public const string GVFSHooksExecutableName = "GVFS.Hooks.exe"; public const string GVFSReadObjectHookExecutableName = "GVFS.ReadObjectHook.exe"; + public const string MountExecutableName = "GVFS.Mount.exe"; public const string GitIsNotInstalledError = "Could not find git.exe. Ensure that Git is installed."; public const string ExecutableExtension = ".exe"; - public static readonly GitVersion MinimumGitVersion = new GitVersion(2, 13, 0, "gvfs", 1, 0); + public static readonly GitVersion MinimumGitVersion = new GitVersion(2, 14, 1, "gvfs", 1, 0); + + public static class GitConfig + { + public const string GVFSPrefix = "gvfs."; + } + + public static class NamedPipes + { + public const int ConnectTimeoutMS = 10000; + } public static class Service { @@ -79,8 +90,11 @@ public static class LogFileTypes public static class DotGVFS { public const string Root = ".gvfs"; + public const string CorruptObjectsName = "CorruptObjects"; + public static readonly string LogPath = Path.Combine(DotGVFS.Root, "logs"); public static readonly string GitObjectCachePath = Path.Combine(DotGVFS.Root, "gitObjectCache"); + public static readonly string CorruptObjectsPath = Path.Combine(DotGVFS.Root, CorruptObjectsName); } public static class DotGit @@ -164,5 +178,24 @@ public static class Heads } } } + + public static class VerbParameters + { + public static class Mount + { + public const string ServiceName = "internal_use_only_service_name"; + public const string Verbosity = "verbosity"; + public const string Keywords = "keywords"; + public const string DebugWindow = "debug-window"; + + public const string DefaultVerbosity = "Informational"; + public const string DefaultKeywords = "Any"; + } + + public static class Unmount + { + public const string SkipLock = "skip-wait-for-lock"; + } + } } } diff --git a/GVFS/GVFS.Common/GVFSContext.cs b/GVFS/GVFS.Common/GVFSContext.cs index adec9e6fa7..36a5b772e9 100644 --- a/GVFS/GVFS.Common/GVFSContext.cs +++ b/GVFS/GVFS.Common/GVFSContext.cs @@ -1,5 +1,5 @@ -using GVFS.Common.Physical.FileSystem; -using GVFS.Common.Physical.Git; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; using GVFS.Common.Tracing; using System; diff --git a/GVFS/GVFS.Common/GVFSEnlistment.cs b/GVFS/GVFS.Common/GVFSEnlistment.cs index 4abbab8a9f..b0927c32fe 100644 --- a/GVFS/GVFS.Common/GVFSEnlistment.cs +++ b/GVFS/GVFS.Common/GVFSEnlistment.cs @@ -1,131 +1,162 @@ using GVFS.Common.Git; -using System; +using GVFS.Common.NamedPipes; +using Newtonsoft.Json; using System.IO; +using System.Security.AccessControl; +using System.Security.Principal; using System.Threading; namespace GVFS.Common { public class GVFSEnlistment : Enlistment { + public const string InvalidRepoUrl = "invalid://repoUrl"; + // New enlistment - public GVFSEnlistment(string enlistmentRoot, string repoUrl, string cacheServerUrl, string gitBinPath, string gvfsHooksRoot) + public GVFSEnlistment(string enlistmentRoot, string repoUrl, string gitBinPath, string gvfsHooksRoot) : base( enlistmentRoot, Path.Combine(enlistmentRoot, GVFSConstants.WorkingDirectoryRootName), Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.GitObjectCachePath), - repoUrl, - cacheServerUrl, + repoUrl, gitBinPath, gvfsHooksRoot) { - this.NamedPipeName = NamedPipes.NamedPipeClient.GetPipeNameFromPath(this.EnlistmentRoot); + this.NamedPipeName = Paths.GetNamedPipeName(this.EnlistmentRoot); this.DotGVFSRoot = Path.Combine(this.EnlistmentRoot, GVFSConstants.DotGVFS.Root); this.GVFSLogsRoot = Path.Combine(this.EnlistmentRoot, GVFSConstants.DotGVFS.LogPath); - - // Mutex name cannot include '\' (other than the '\' after Global) - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms682411(v=vs.85).aspx - this.EnlistmentMutex = new Mutex(false, "Global\\" + this.NamedPipeName.Replace('\\', ':')); } - - // Enlistment without repo url. This skips git commands that may fail in a corrupt repo. - public GVFSEnlistment(string enlistmentRoot, string gitBinPath) - : this( - enlistmentRoot, - repoUrl: "invalid://repoUrl", - cacheServerUrl: "invalid://cacheServerUrl", - gitBinPath: gitBinPath, - gvfsHooksRoot: null) - { - } - + // Existing, configured enlistment - public GVFSEnlistment(string enlistmentRoot, string cacheServerUrl, string gitBinPath, string gvfsHooksRoot) + public GVFSEnlistment(string enlistmentRoot, string gitBinPath, string gvfsHooksRoot) : this( - enlistmentRoot, + enlistmentRoot, null, - cacheServerUrl, - gitBinPath, + gitBinPath, gvfsHooksRoot) { } - public Mutex EnlistmentMutex { get; } - public string NamedPipeName { get; private set; } public string DotGVFSRoot { get; private set; } public string GVFSLogsRoot { get; private set; } - public static GVFSEnlistment CreateFromCurrentDirectory(string cacheServerUrl, string gitBinRoot) - { - return CreateFromDirectory(Environment.CurrentDirectory, cacheServerUrl, gitBinRoot, null); - } - - public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, string gitBinRoot) + public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, string gitBinRoot, string gvfsHooksRoot) { if (Directory.Exists(directory)) { - string enlistmentRoot = EnlistmentUtils.GetEnlistmentRoot(directory); + string enlistmentRoot = Paths.GetGVFSEnlistmentRoot(directory); if (enlistmentRoot != null) { - return new GVFSEnlistment(enlistmentRoot, gitBinRoot); + return new GVFSEnlistment(enlistmentRoot, InvalidRepoUrl, gitBinRoot, gvfsHooksRoot); } } return null; } - public static GVFSEnlistment CreateFromDirectory(string directory, string cacheServerUrl, string gitBinRoot, string gvfsHooksRoot) + public static GVFSEnlistment CreateFromDirectory(string directory, string gitBinRoot, string gvfsHooksRoot) { if (Directory.Exists(directory)) { - string enlistmentRoot = EnlistmentUtils.GetEnlistmentRoot(directory); + string enlistmentRoot = Paths.GetGVFSEnlistmentRoot(directory); if (enlistmentRoot != null) { - return new GVFSEnlistment(enlistmentRoot, cacheServerUrl, gitBinRoot, gvfsHooksRoot); + return new GVFSEnlistment(enlistmentRoot, gitBinRoot, gvfsHooksRoot); } } return null; } - public static string ToFullPath(string originalValue, string toUseIfOriginalNullOrWhitespace) - { - if (string.IsNullOrWhiteSpace(originalValue)) - { - return toUseIfOriginalNullOrWhitespace; - } - - try - { - return Path.GetFullPath(originalValue); - } - catch (Exception) - { - return null; - } - } - public static string GetNewGVFSLogFileName(string logsRoot, string logFileType) { return Enlistment.GetNewLogFileName( logsRoot, "gvfs_" + logFileType); } - - public bool TrySetCacheServerUrlConfig() + + public static bool WaitUntilMounted(string enlistmentRoot, out string errorMessage) { - GitProcess git = new Git.GitProcess(this); - string settingName = Enlistment.GetCacheConfigSettingName(this.RepoUrl); - return !git.SetInLocalConfig(settingName, this.CacheServerUrl).HasErrors; - } + errorMessage = null; + using (NamedPipeClient pipeClient = new NamedPipeClient(NamedPipeClient.GetPipeNameFromPath(enlistmentRoot))) + { + if (!pipeClient.Connect(GVFSConstants.NamedPipes.ConnectTimeoutMS)) + { + errorMessage = "Unable to mount because the GVFS.Mount process is not responding."; + return false; + } + while (true) + { + string response = string.Empty; + try + { + pipeClient.SendRequest(NamedPipeMessages.GetStatus.Request); + response = pipeClient.ReadRawResponse(); + NamedPipeMessages.GetStatus.Response getStatusResponse = + NamedPipeMessages.GetStatus.Response.FromJson(response); + + if (getStatusResponse.MountStatus == NamedPipeMessages.GetStatus.Ready) + { + return true; + } + else if (getStatusResponse.MountStatus == NamedPipeMessages.GetStatus.MountFailed) + { + errorMessage = string.Format("Failed to mount at {0}", enlistmentRoot); + return false; + } + else + { + Thread.Sleep(500); + } + } + catch (BrokenPipeException e) + { + errorMessage = string.Format("Could not connect to GVFS.Mount: {0}", e); + return false; + } + catch (JsonReaderException e) + { + errorMessage = string.Format("Failed to parse response from GVFS.Mount.\n {0}", e); + return false; + } + } + } + } + public bool TryCreateEnlistmentFolders() { try { Directory.CreateDirectory(this.EnlistmentRoot); + + // The following permissions are typically present on deskop and missing on Server + // + // ACCESS_ALLOWED_ACE_TYPE: NT AUTHORITY\Authenticated Users + // [OBJECT_INHERIT_ACE] + // [CONTAINER_INHERIT_ACE] + // [INHERIT_ONLY_ACE] + // DELETE + // GENERIC_EXECUTE + // GENERIC_WRITE + // GENERIC_READ + DirectorySecurity rootSecurity = Directory.GetAccessControl(this.EnlistmentRoot); + AccessRule authenticatedUsersAccessRule = rootSecurity.AccessRuleFactory( + new SecurityIdentifier(WellKnownSidType.AuthenticatedUserSid, null), + unchecked((int)(NativeMethods.FileAccess.DELETE | NativeMethods.FileAccess.GENERIC_EXECUTE | NativeMethods.FileAccess.GENERIC_WRITE | NativeMethods.FileAccess.GENERIC_READ)), + true, + InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, + PropagationFlags.None, + AccessControlType.Allow); + + // The return type of the AccessRuleFactory method is the base class, AccessRule, but the return value can be cast safely to the derived class. + // https://msdn.microsoft.com/en-us/library/system.security.accesscontrol.filesystemsecurity.accessrulefactory(v=vs.110).aspx + rootSecurity.AddAccessRule((FileSystemAccessRule)authenticatedUsersAccessRule); + Directory.SetAccessControl(this.EnlistmentRoot, rootSecurity); + Directory.CreateDirectory(this.WorkingDirectoryRoot); this.CreateHiddenDirectory(this.DotGVFSRoot); } diff --git a/GVFS/GVFS.Common/GVFSLock.Shared.cs b/GVFS/GVFS.Common/GVFSLock.Shared.cs new file mode 100644 index 0000000000..831c21e850 --- /dev/null +++ b/GVFS/GVFS.Common/GVFSLock.Shared.cs @@ -0,0 +1,121 @@ +using GVFS.Common.NamedPipes; +using System; +using System.Diagnostics; +using System.Threading; + +namespace GVFS.Common +{ + // This file contains methods that are used by GVFS.Hooks (compiled both by GVFS.Common and GVFS.Hooks). + public partial class GVFSLock + { + public static bool TryAcquireGVFSLockForProcess( + NamedPipeClient pipeClient, + string fullCommand, + int pid, + Process parentProcess, + string gvfsEnlistmentRoot, + out string result) + { + NamedPipeMessages.LockRequest request = + new NamedPipeMessages.LockRequest(pid, fullCommand); + + NamedPipeMessages.Message requestMessage = request.CreateMessage(NamedPipeMessages.AcquireLock.AcquireRequest); + pipeClient.SendRequest(requestMessage); + + NamedPipeMessages.AcquireLock.Response response = new NamedPipeMessages.AcquireLock.Response(pipeClient.ReadResponse()); + + if (response.Result == NamedPipeMessages.AcquireLock.AcceptResult) + { + result = null; + return true; + } + else if (response.Result == NamedPipeMessages.AcquireLock.MountNotReadyResult) + { + result = "GVFS has not finished initializing, please wait a few seconds and try again."; + return false; + } + else + { + string message = string.Empty; + switch (response.Result) + { + case NamedPipeMessages.AcquireLock.AcceptResult: + break; + + case NamedPipeMessages.AcquireLock.DenyGVFSResult: + message = "Waiting for GVFS to release the lock"; + break; + + case NamedPipeMessages.AcquireLock.DenyGitResult: + message = string.Format("Waiting for '{0}' to release the lock", response.ResponseData.ParsedCommand); + break; + + default: + result = "Error when acquiring the lock. Unrecognized response: " + response.CreateMessage(); + return false; + } + + ConsoleHelper.ShowStatusWhileRunning( + () => + { + while (response.Result != NamedPipeMessages.AcquireLock.AcceptResult) + { + Thread.Sleep(250); + pipeClient.SendRequest(requestMessage); + response = new NamedPipeMessages.AcquireLock.Response(pipeClient.ReadResponse()); + } + + return true; + }, + message, + output: Console.Out, + showSpinner: !ConsoleHelper.IsConsoleOutputRedirectedToFile(), + gvfsLogEnlistmentRoot: gvfsEnlistmentRoot); + + result = null; + return true; + } + } + + public static void ReleaseGVFSLock( + NamedPipeClient pipeClient, + string fullCommand, + int pid, + Process parentProcess, + Action responseHandler, + string gvfsEnlistmentRoot, + string waitingMessage = "", + int spinnerDelay = 0) + { + NamedPipeMessages.LockRequest request = + new NamedPipeMessages.LockRequest(pid, fullCommand); + + NamedPipeMessages.Message requestMessage = request.CreateMessage(NamedPipeMessages.ReleaseLock.Request); + + pipeClient.SendRequest(requestMessage); + NamedPipeMessages.ReleaseLock.Response response = null; + + if (ConsoleHelper.IsConsoleOutputRedirectedToFile()) + { + // If output is redirected then don't show waiting message or it might be interpreted as error + response = new NamedPipeMessages.ReleaseLock.Response(pipeClient.ReadResponse()); + responseHandler(response); + } + else + { + ConsoleHelper.ShowStatusWhileRunning( + () => + { + response = new NamedPipeMessages.ReleaseLock.Response(pipeClient.ReadResponse()); + responseHandler(response); + return ConsoleHelper.ActionResult.Success; + }, + waitingMessage, + output: Console.Out, + showSpinner: true, + gvfsLogEnlistmentRoot: gvfsEnlistmentRoot, + initialDelayMs: spinnerDelay); + } + } + } +} diff --git a/GVFS/GVFS.Common/GVFSLock.cs b/GVFS/GVFS.Common/GVFSLock.cs index a94707ad10..3063fa5833 100644 --- a/GVFS/GVFS.Common/GVFSLock.cs +++ b/GVFS/GVFS.Common/GVFSLock.cs @@ -7,14 +7,12 @@ namespace GVFS.Common { - public class GVFSLock : IDisposable + public partial class GVFSLock : IDisposable { - private const string CommandParentExePrefix = "git"; - private readonly object acquisitionLock = new object(); private readonly ITracer tracer; - private readonly ProcessWatcher processWatcher; + private ProcessWatcher processWatcher; private NamedPipeMessages.LockData lockHolder; private ManualResetEvent externalLockReleased; @@ -77,7 +75,7 @@ public bool IsLockedByGVFS Process process; if (ProcessHelper.TryGetProcess(requester.PID, out process)) { - this.processWatcher.WatchForTermination(requester.PID, GVFSLock.CommandParentExePrefix); + this.processWatcher.WatchForTermination(requester.PID); process.Dispose(); this.lockHolder = requester; @@ -152,10 +150,7 @@ public bool TryAcquireLock() public void ReleaseLock() { this.tracer.RelatedEvent(EventLevel.Verbose, "ReleaseLock", new EventMetadata()); - lock (this.acquisitionLock) - { - this.IsLockedByGVFS = false; - } + this.IsLockedByGVFS = false; } public bool ReleaseExternalLock(int pid) @@ -168,27 +163,9 @@ public bool WaitOnExternalLockRelease(int millisecondsTimeout) return this.externalLockReleased.WaitOne(millisecondsTimeout); } - /// - /// Returns true if the lock is currently held by an external - /// caller that represents a git call using one of the specified git verbs. - /// - public bool IsLockedByGitVerb(params string[] verbs) - { - string command = this.GetLockedGitCommand(); - if (!string.IsNullOrEmpty(command)) - { - return GitHelper.IsVerb(command, verbs); - } - - return false; - } - public bool IsExternalLockHolderAlive() { - lock (this.acquisitionLock) - { - return this.lockHolder != null; - } + return this.lockHolder != null; } public string GetLockedGitCommand() @@ -231,6 +208,12 @@ protected void Dispose(bool disposing) { if (disposing) { + if (this.processWatcher != null) + { + this.processWatcher.Dispose(); + this.processWatcher = null; + } + if (this.externalLockReleased != null) { this.externalLockReleased.Dispose(); diff --git a/GVFS/GVFS.Common/Physical/Git/GVFSGitObjects.cs b/GVFS/GVFS.Common/Git/GVFSGitObjects.cs similarity index 52% rename from GVFS/GVFS.Common/Physical/Git/GVFSGitObjects.cs rename to GVFS/GVFS.Common/Git/GVFSGitObjects.cs index 140c754b7d..00b4fb998e 100644 --- a/GVFS/GVFS.Common/Physical/Git/GVFSGitObjects.cs +++ b/GVFS/GVFS.Common/Git/GVFSGitObjects.cs @@ -1,12 +1,11 @@ -using GVFS.Common.Git; -using GVFS.Common.Http; +using GVFS.Common.Http; +using GVFS.Common.Tracing; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; -using System.Threading; -namespace GVFS.Common.Physical.Git +namespace GVFS.Common.Git { public class GVFSGitObjects : GitObjects { @@ -25,32 +24,53 @@ public GVFSGitObjects(GVFSContext context, GitObjectsHttpRequestor objectRequest public bool TryCopyBlobContentStream(string sha, Action writeAction) { - if (!this.Context.Repository.TryCopyBlobContentStream(sha, writeAction)) - { - if (!this.TryDownloadAndSaveObject(sha.Substring(0, 2), sha.Substring(2))) + RetryWrapper retrier = new RetryWrapper(this.GitObjectRequestor.RetryConfig.MaxAttempts); + retrier.OnFailure += + errorArgs => { - return false; - } + EventMetadata metadata = new EventMetadata(); + metadata.Add("sha", sha); + metadata.Add("AttemptNumber", errorArgs.TryCount); + metadata.Add("WillRetry", errorArgs.WillRetry); + metadata.Add("ErrorMessage", "TryCopyBlobContentStream: Failed to provide blob contents"); + this.Tracer.RelatedError(metadata); + }; - if (!this.Context.Repository.TryCopyBlobContentStream(sha, writeAction)) - { - this.Tracer.RelatedError("Failed to cat-file after download. Trying again: " + sha); + string firstTwoShaDigits = sha.Substring(0, 2); + string remainingShaDigits = sha.Substring(2); - // Due to a potential race, git sometimes fail to read the blob even though we just wrote it out. - // Retrying the read fixes that issue. - Thread.Sleep(100); - if (!this.Context.Repository.TryCopyBlobContentStream(sha, writeAction)) + RetryWrapper.InvocationResult invokeResult = retrier.Invoke( + tryCount => + { + bool success = this.Context.Repository.TryCopyBlobContentStream(sha, writeAction); + if (success) { - this.Tracer.RelatedError("Failed to cat-file after multiple attempts: " + sha); - return false; + return new RetryWrapper.CallbackResult(true); } - } - } + else + { + // Pass in 1 for maxAttempts because the retrier in this method manages multiple attempts + if (this.TryDownloadAndSaveObject(firstTwoShaDigits, remainingShaDigits, maxAttempts: 1)) + { + if (this.Context.Repository.TryCopyBlobContentStream(sha, writeAction)) + { + return new RetryWrapper.CallbackResult(true); + } + } + + return new RetryWrapper.CallbackResult(error: null, shouldRetry: true); + } + }); - return true; + return invokeResult.Result; } public bool TryDownloadAndSaveObject(string firstTwoShaDigits, string remainingShaDigits) + { + return this.TryDownloadAndSaveObject(firstTwoShaDigits, remainingShaDigits, GitObjectsHttpRequestor.UseConfiguredMaxAttempts); + } + + public bool TryDownloadAndSaveObject(string firstTwoShaDigits, string remainingShaDigits, int maxAttempts) { DateTime negativeCacheRequestTime; string objectId = firstTwoShaDigits + remainingShaDigits; @@ -65,7 +85,7 @@ public bool TryDownloadAndSaveObject(string firstTwoShaDigits, string remainingS this.objectNegativeCache.TryRemove(objectId, out negativeCacheRequestTime); } - DownloadAndSaveObjectResult result = this.TryDownloadAndSaveObject(objectId); + DownloadAndSaveObjectResult result = this.TryDownloadAndSaveObject(objectId, maxAttempts); switch (result) { diff --git a/GVFS/GVFS.Common/Git/GitAuthentication.cs b/GVFS/GVFS.Common/Git/GitAuthentication.cs index 34df2d490d..59a3020dae 100644 --- a/GVFS/GVFS.Common/Git/GitAuthentication.cs +++ b/GVFS/GVFS.Common/Git/GitAuthentication.cs @@ -6,9 +6,9 @@ namespace GVFS.Common.Git { public class GitAuthentication { + private const double MaxBackoffSeconds = 30; private readonly object gitAuthLock = new object(); - - private int numberOfRetries = -1; + private int numberOfAttempts = 0; private DateTime lastAuthAttempt = DateTime.MinValue; private string cachedAuthString; @@ -25,26 +25,34 @@ public GitAuthentication(GitProcess git) this.git = git; } + public bool IsBackingOff + { + get + { + return this.GetNextAuthAttemptTime() > DateTime.Now; + } + } + public void ConfirmCredentialsWorked(string usedCredential) { lock (this.gitAuthLock) { if (usedCredential == this.cachedAuthString) { - this.numberOfRetries = -1; + this.numberOfAttempts = 0; this.lastAuthAttempt = DateTime.MinValue; } } } - public bool RevokeAndCheckCanRetry(string usedCredential) + public void Revoke(string usedCredential) { lock (this.gitAuthLock) { if (usedCredential != this.cachedAuthString) { // Don't stomp a different credential - return true; + return; } if (this.cachedAuthString != null) @@ -55,8 +63,6 @@ public bool RevokeAndCheckCanRetry(string usedCredential) this.git.RevokeCredential(); this.UpdateBackoff(); } - - return !this.IsBackingOff(); } } @@ -78,37 +84,31 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri string gitUsername; string gitPassword; - // These auth settings are necessary to support running the functional tests on build servers. - // The reason it's needed is that the GVFS.Service runs as LocalSystem, and the build agent does not - // so storing the agent's creds in the Windows Credential Store does not allow the service to discover it - GitProcess.Result usernameResult = this.git.GetFromConfig("gvfs.FunctionalTests.UserName"); - GitProcess.Result passwordResult = this.git.GetFromConfig("gvfs.FunctionalTests.Password"); + if (this.IsBackingOff) + { + gitAuthString = null; + errorMessage = "Auth failed. No retries will be made until: " + this.GetNextAuthAttemptTime(); + + // TODO 1026787: Log to event log if numberOfRetries >=3 + return false; + } - if (!usernameResult.HasErrors && - !passwordResult.HasErrors) + if (!this.git.TryGetCredentials(tracer, out gitUsername, out gitPassword)) { - gitUsername = usernameResult.Output.TrimEnd('\n'); - gitPassword = passwordResult.Output.TrimEnd('\n'); + gitAuthString = null; + errorMessage = "Authentication failed."; + this.UpdateBackoff(); + return false; + } + + if (!string.IsNullOrEmpty(gitUsername) && !string.IsNullOrEmpty(gitPassword)) + { + this.cachedAuthString = Convert.ToBase64String(Encoding.ASCII.GetBytes(gitUsername + ":" + gitPassword)); } else { - if (this.IsBackingOff()) - { - gitAuthString = null; - errorMessage = "Auth failed. No retries will be made until: " + this.GetNextAuthAttemptTime(); - return false; - } - - if (!this.git.TryGetCredentials(tracer, out gitUsername, out gitPassword)) - { - gitAuthString = null; - errorMessage = "Authentication failed."; - this.UpdateBackoff(); - return false; - } + this.cachedAuthString = string.Empty; } - - this.cachedAuthString = Convert.ToBase64String(Encoding.ASCII.GetBytes(gitUsername + ":" + gitPassword)); } gitAuthString = this.cachedAuthString; @@ -119,29 +119,21 @@ public bool TryGetCredentials(ITracer tracer, out string gitAuthString, out stri return true; } - private bool IsBackingOff() - { - return this.GetNextAuthAttemptTime() > DateTime.Now; - } - private DateTime GetNextAuthAttemptTime() { - switch (this.numberOfRetries) + if (this.numberOfAttempts <= 1) { - case -1: - case 0: - return DateTime.MinValue; - case 1: - return this.lastAuthAttempt + TimeSpan.FromSeconds(30); - default: - return DateTime.MaxValue; + return DateTime.MinValue; } + + double backoffSeconds = RetryBackoff.CalculateBackoffSeconds(this.numberOfAttempts, MaxBackoffSeconds); + return this.lastAuthAttempt + TimeSpan.FromSeconds(backoffSeconds); } private void UpdateBackoff() { this.lastAuthAttempt = DateTime.Now; - this.numberOfRetries++; + this.numberOfAttempts++; } } } diff --git a/GVFS/GVFS.Common/Git/GitConfigHelper.cs b/GVFS/GVFS.Common/Git/GitConfigHelper.cs index 4c7eae5448..28e3a053f1 100644 --- a/GVFS/GVFS.Common/Git/GitConfigHelper.cs +++ b/GVFS/GVFS.Common/Git/GitConfigHelper.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; namespace GVFS.Common.Git { @@ -34,12 +33,11 @@ public static bool TrySanitizeConfigFileLine(string fileLine, out string sanitiz /// /// Get the settings for a section in a given config file. /// - /// The path to the config file. + /// The contents of a config file, one line per entry. /// The name of the section to grab the settings from. /// A dictionary of settings, keyed off the setting name. - public static Dictionary GetSettings(string configFilename, string sectionName) + public static Dictionary GetSettings(string[] configLines, string sectionName) { - string[] configLines = File.ReadAllLines(configFilename); List linesToParse = new List(); int currentLineIndex = 0; diff --git a/GVFS/GVFS.Common/Git/GitObjects.cs b/GVFS/GVFS.Common/Git/GitObjects.cs index 4dc750250e..c1a574ed4e 100644 --- a/GVFS/GVFS.Common/Git/GitObjects.cs +++ b/GVFS/GVFS.Common/Git/GitObjects.cs @@ -1,4 +1,5 @@ using GVFS.Common.Http; +using GVFS.Common.NetworkStreams; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using System; @@ -34,9 +35,9 @@ public enum DownloadAndSaveObjectResult Error } - public virtual bool TryDownloadAndSaveCommits(IEnumerable commitShas, int commitDepth) + public virtual bool TryDownloadAndSaveCommit(string commitSha, int commitDepth) { - return this.TryDownloadAndSaveObjects(commitShas, commitDepth, preferLooseObjects: false); + return this.TryDownloadAndSaveObjects(new string[] { commitSha }, commitDepth, preferLooseObjects: false); } public bool TryDownloadAndSaveBlobs(IEnumerable blobShas) @@ -60,7 +61,7 @@ public bool TryDownloadPrefetchPacks(long latestTimestamp) endPointGenerator: () => new Uri( string.Format( "{0}?lastPackTimestamp={1}", - this.Enlistment.PrefetchEndpointUrl, + this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl, latestTimestamp)), requestBodyGenerator: () => null, acceptType: new MediaTypeWithQualityHeaderValue(GVFSConstants.MediaTypes.PrefetchPackFilesAndIndexesMediaType)); @@ -71,7 +72,7 @@ public bool TryDownloadPrefetchPacks(long latestTimestamp) { EventMetadata warning = new EventMetadata(); warning.Add("ErrorMessage", "The server does not support " + GVFSConstants.Endpoints.GVFSPrefetch); - warning.Add(nameof(this.Enlistment.PrefetchEndpointUrl), this.Enlistment.PrefetchEndpointUrl); + warning.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); activity.RelatedEvent(EventLevel.Warning, "CommandNotSupported", warning); } else @@ -80,7 +81,7 @@ public bool TryDownloadPrefetchPacks(long latestTimestamp) error.Add("latestTimestamp", latestTimestamp); error.Add("Exception", result.Error); error.Add("ErrorMessage", "DownloadPrefetchPacks failed."); - error.Add(nameof(this.Enlistment.PrefetchEndpointUrl), this.Enlistment.PrefetchEndpointUrl); + error.Add(nameof(this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl), this.GitObjectRequestor.CacheServer.PrefetchEndpointUrl); activity.RelatedError(error); } } @@ -203,7 +204,7 @@ public virtual string[] ReadPackFileNames(string prefixFilter = "") return Directory.GetFiles(this.Enlistment.GitPackRoot, prefixFilter + "*.pack"); } - protected virtual DownloadAndSaveObjectResult TryDownloadAndSaveObject(string objectSha) + protected virtual DownloadAndSaveObjectResult TryDownloadAndSaveObject(string objectSha, int maxAttempts) { if (objectSha == GVFSConstants.AllZeroSha) { @@ -215,6 +216,7 @@ protected virtual DownloadAndSaveObjectResult TryDownloadAndSaveObject(string ob RetryWrapper.InvocationResult output = this.GitObjectRequestor.TryDownloadLooseObject( objectSha, + maxAttempts, onSuccess: (tryCount, response) => { this.WriteLooseObject(response.Stream, objectSha, bufToCopyWith); diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index c3f2c13d4c..5750dc60ec 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -1,4 +1,3 @@ -using GVFS.Common.Physical; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using Microsoft.Win32; @@ -19,12 +18,22 @@ public class GitProcess private const string GitInstallationRegistryKey = "SOFTWARE\\GitForWindows"; private const string GitInstallationRegistryInstallPathValue = "InstallPath"; - private static readonly Encoding StandardEncoding = Encoding.GetEncoding(437); + private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false); private object executionLock = new object(); private Enlistment enlistment; + static GitProcess() + { + // If the encoding is UTF8, .Net's default behavior will include a BOM + // We need to use the BOM-less encoding because Git doesn't understand it + if (Console.InputEncoding.CodePage == UTF8NoBOM.CodePage) + { + Console.InputEncoding = UTF8NoBOM; + } + } + public GitProcess(Enlistment enlistment) { if (enlistment == null) @@ -52,7 +61,7 @@ public static bool GitExists(string gitBinPath) public static string GetInstalledGitBinPath() { - string gitBinPath = RegistryUtils.GetStringFromRegistry(RegistryHive.LocalMachine, GitInstallationRegistryKey, GitInstallationRegistryInstallPathValue); + string gitBinPath = ProcessHelper.GetStringFromRegistry(RegistryHive.LocalMachine, GitInstallationRegistryKey, GitInstallationRegistryInstallPathValue); if (!string.IsNullOrWhiteSpace(gitBinPath)) { gitBinPath = Path.Combine(gitBinPath, GitBinRelativePath); @@ -93,7 +102,7 @@ public virtual void RevokeCredential() using (ITracer activity = tracer.StartActivity("TryGetCredentials", EventLevel.Informational)) { - Result gitCredentialOutput = this.InvokeGitOutsideEnlistment( + Result gitCredentialOutput = this.InvokeGitAgainstDotGitFolder( "credential fill", stdin => stdin.Write("url=" + this.enlistment.RepoUrl + "\n\n"), parseStdOutLine: null); @@ -184,7 +193,7 @@ public bool TryGetAllLocalConfig(out Dictionary config /// otherwise it will run it from outside the enlistment. /// /// The value found for the setting. - public Result GetFromConfig(string settingName, bool forceOutsideEnlistment = false) + public virtual Result GetFromConfig(string settingName, bool forceOutsideEnlistment = false) { string command = string.Format("config {0}", settingName); @@ -226,28 +235,12 @@ public bool TryGetFromConfig(string settingName, bool forceOutsideEnlistment, ou public Result GetOriginUrl() { - Result result = this.InvokeGitAgainstDotGitFolder("remote -v"); - if (result.HasErrors) - { - return result; - } - - string[] lines = result.Output.Split('\r', '\n'); - string originFetchLine = lines.Where( - l => l.StartsWith("origin", StringComparison.OrdinalIgnoreCase) - && l.EndsWith("(fetch)")).FirstOrDefault(); - if (originFetchLine == null) - { - throw new InvalidRepoException("remote 'origin' is not configured for this repo"); - } - - string[] parts = originFetchLine.Split('\t', ' '); - return new Result(parts[1], string.Empty, 0); + return this.InvokeGitAgainstDotGitFolder("config --local remote.origin.url"); } - public void DiffTree(string sourceTreeish, string targetTreeish, Action onResult) + public Result DiffTree(string sourceTreeish, string targetTreeish, Action onResult) { - this.InvokeGitAgainstDotGitFolder("diff-tree -r -t " + sourceTreeish + " " + targetTreeish, null, onResult); + return this.InvokeGitAgainstDotGitFolder("diff-tree -r -t " + sourceTreeish + " " + targetTreeish, null, onResult); } public Result CreateBranchWithUpstream(string branchToCreate, string upstreamBranch) @@ -344,13 +337,13 @@ public Process GetGitProcess(string command, string workingDirectory, string dot processInfo.WindowStyle = ProcessWindowStyle.Hidden; processInfo.CreateNoWindow = true; - processInfo.StandardOutputEncoding = StandardEncoding; - processInfo.StandardErrorEncoding = StandardEncoding; + processInfo.StandardOutputEncoding = UTF8NoBOM; + processInfo.StandardErrorEncoding = UTF8NoBOM; // Removing trace variables that might change git output and break parsing // List of environment variables: https://git-scm.com/book/gr/v2/Git-Internals-Environment-Variables foreach (string key in processInfo.EnvironmentVariables.Keys.Cast() - .Where(x => x.StartsWith("GIT_TRACE")) + .Where(x => x.StartsWith("GIT_TRACE", StringComparison.OrdinalIgnoreCase)) .ToList()) { processInfo.EnvironmentVariables.Remove(key); @@ -380,6 +373,75 @@ public Process GetGitProcess(string command, string workingDirectory, string dot return executingProcess; } + protected virtual Result InvokeGitImpl( + string command, + string workingDirectory, + string dotGitDirectory, + bool useReadObjectHook, + Action writeStdIn, + Action parseStdOutLine, + int timeoutMs) + { + try + { + // From https://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx + // To avoid deadlocks, use asynchronous read operations on at least one of the streams. + // Do not perform a synchronous read to the end of both redirected streams. + using (Process executingProcess = this.GetGitProcess(command, workingDirectory, dotGitDirectory, useReadObjectHook, redirectStandardError: true)) + { + StringBuilder output = new StringBuilder(); + StringBuilder errors = new StringBuilder(); + + executingProcess.ErrorDataReceived += (sender, args) => + { + if (args.Data != null) + { + errors.Append(args.Data + "\n"); + } + }; + executingProcess.OutputDataReceived += (sender, args) => + { + if (args.Data != null) + { + if (parseStdOutLine != null) + { + parseStdOutLine(args.Data); + } + else + { + output.Append(args.Data + "\n"); + } + } + }; + + lock (this.executionLock) + { + executingProcess.Start(); + + if (writeStdIn != null) + { + writeStdIn(executingProcess.StandardInput); + } + + executingProcess.BeginOutputReadLine(); + executingProcess.BeginErrorReadLine(); + + if (!executingProcess.WaitForExit(timeoutMs)) + { + executingProcess.Kill(); + return new Result(output.ToString(), "Operation timed out: " + errors.ToString(), Result.GenericFailureCode); + } + } + + return new Result(output.ToString(), errors.ToString(), executingProcess.ExitCode); + } + } + catch (Win32Exception e) + { + return new Result(string.Empty, e.Message, Result.GenericFailureCode); + } + } + private static string ParseValue(string contents, string prefix) { int startIndex = contents.IndexOf(prefix) + prefix.Length; @@ -403,7 +465,7 @@ private static string ParseValue(string contents, string prefix) /// /// /// For commands where git doesn't need to be (or can't be) run from inside an enlistment. - /// eg. 'git init' or 'git credential' + /// eg. 'git init' or 'git version' /// private Result InvokeGitOutsideEnlistment(string command) { @@ -467,76 +529,7 @@ private Result InvokeGitAgainstDotGitFolder(string command) parseStdOutLine: parseStdOutLine, timeoutMs: -1); } - - private Result InvokeGitImpl( - string command, - string workingDirectory, - string dotGitDirectory, - bool useReadObjectHook, - Action writeStdIn, - Action parseStdOutLine, - int timeoutMs) - { - try - { - // From https://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx - // To avoid deadlocks, use asynchronous read operations on at least one of the streams. - // Do not perform a synchronous read to the end of both redirected streams. - using (Process executingProcess = this.GetGitProcess(command, workingDirectory, dotGitDirectory, useReadObjectHook, redirectStandardError: true)) - { - StringBuilder output = new StringBuilder(); - StringBuilder errors = new StringBuilder(); - - executingProcess.ErrorDataReceived += (sender, args) => - { - if (args.Data != null) - { - errors.Append(args.Data + "\n"); - } - }; - executingProcess.OutputDataReceived += (sender, args) => - { - if (args.Data != null) - { - if (parseStdOutLine != null) - { - parseStdOutLine(args.Data); - } - else - { - output.Append(args.Data + "\n"); - } - } - }; - - lock (this.executionLock) - { - executingProcess.Start(); - - if (writeStdIn != null) - { - writeStdIn(executingProcess.StandardInput); - } - - executingProcess.BeginOutputReadLine(); - executingProcess.BeginErrorReadLine(); - - if (!executingProcess.WaitForExit(timeoutMs)) - { - executingProcess.Kill(); - return new Result(output.ToString(), "Operation timed out: " + errors.ToString(), Result.GenericFailureCode); - } - } - - return new Result(output.ToString(), errors.ToString(), executingProcess.ExitCode); - } - } - catch (Win32Exception e) - { - return new Result(string.Empty, e.Message, Result.GenericFailureCode); - } - } - + public class Result { public const int SuccessCode = 0; diff --git a/GVFS/GVFS.Common/Git/GitRefs.cs b/GVFS/GVFS.Common/Git/GitRefs.cs index 37595a9c6c..11cdf0d363 100644 --- a/GVFS/GVFS.Common/Git/GitRefs.cs +++ b/GVFS/GVFS.Common/Git/GitRefs.cs @@ -52,9 +52,9 @@ public int Count get { return this.commitsPerRef.Count; } } - public IEnumerable GetTipCommitIds() + public string GetTipCommitId(string branch) { - return this.commitsPerRef.Values; + return this.commitsPerRef[OriginRemoteRefPrefix + branch]; } public string GetDefaultBranch() diff --git a/GVFS/GVFS.Common/Physical/Git/GitRepo.cs b/GVFS/GVFS.Common/Git/GitRepo.cs similarity index 61% rename from GVFS/GVFS.Common/Physical/Git/GitRepo.cs rename to GVFS/GVFS.Common/Git/GitRepo.cs index 426274e253..3fc7cc19d6 100644 --- a/GVFS/GVFS.Common/Physical/Git/GitRepo.cs +++ b/GVFS/GVFS.Common/Git/GitRepo.cs @@ -1,12 +1,12 @@ -using GVFS.Common.Git; -using GVFS.Common.Physical.FileSystem; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; +using Microsoft.Diagnostics.Tracing; using System; using System.IO; using System.IO.Compression; using System.Linq; -namespace GVFS.Common.Physical.Git +namespace GVFS.Common.Git { public class GitRepo : IDisposable { @@ -48,7 +48,8 @@ public virtual bool TryCopyBlobContentStream(string blobSha, Action repo.TryCopyBlob(blobSha, writeAction), out copyBlobResult)) diff --git a/GVFS/GVFS.Common/Git/GitTreeEntry.cs b/GVFS/GVFS.Common/Git/GitTreeEntry.cs deleted file mode 100644 index 208196ba96..0000000000 --- a/GVFS/GVFS.Common/Git/GitTreeEntry.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace GVFS.Common.Git -{ - public class GitTreeEntry - { - public GitTreeEntry(string name, string sha, bool isTree, bool isBlob) - { - this.Name = name; - this.Sha = sha; - this.IsTree = isTree; - this.IsBlob = isBlob; - } - - public string Name { get; private set; } - public string Sha { get; private set; } - public bool IsTree { get; private set; } - public bool IsBlob { get; private set; } - - public long Size { get; set; } - } -} diff --git a/GVFS/GVFS.Common/Git/LibGit2Helpers.cs b/GVFS/GVFS.Common/Git/LibGit2Helpers.cs deleted file mode 100644 index 2ba4477ef1..0000000000 --- a/GVFS/GVFS.Common/Git/LibGit2Helpers.cs +++ /dev/null @@ -1,94 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace GVFS.Common.Git -{ - public static class LibGit2Helpers - { - public const uint SuccessCode = 0; - - public const string Git2DllName = "git2.dll"; - - public enum ObjectTypes - { - Commit = 1, - Tree = 2, - Blob = 3, - } - - public static GitOid IntPtrToGitOid(IntPtr oidPtr) - { - return Marshal.PtrToStructure(oidPtr); - } - - [DllImport(Git2DllName, EntryPoint = "git_libgit2_init")] - public static extern void Init(); - - [DllImport(Git2DllName, EntryPoint = "git_libgit2_shutdown")] - public static extern int Shutdown(); - - [DllImport(Git2DllName, EntryPoint = "git_revparse_single")] - public static extern uint RevParseSingle(out IntPtr objectHandle, IntPtr repoHandle, string oid); - - [DllImport(Git2DllName, EntryPoint = "git_oid_fromstr")] - public static extern void OidFromString(ref GitOid oid, string hash); - - public static string GetLastError() - { - IntPtr ptr = GetLastGitError(); - if (ptr == IntPtr.Zero) - { - return "Operation was successful"; - } - - return Marshal.PtrToStructure(ptr).Message; - } - - [DllImport(Git2DllName, EntryPoint = "giterr_last")] - private static extern IntPtr GetLastGitError(); - - [StructLayout(LayoutKind.Sequential)] - private struct GitError - { - [MarshalAs(UnmanagedType.LPStr)] - public string Message; - - public int Klass; - } - - public static class Repo - { - [DllImport(Git2DllName, EntryPoint = "git_repository_open")] - public static extern uint Open(out IntPtr repoHandle, string path); - - [DllImport(Git2DllName, EntryPoint = "git_tree_free")] - public static extern void Free(IntPtr repoHandle); - } - - public static class Object - { - [DllImport(Git2DllName, EntryPoint = "git_object_type")] - public static extern ObjectTypes GetType(IntPtr objectHandle); - - [DllImport(Git2DllName, EntryPoint = "git_object_free")] - public static extern void Free(IntPtr objHandle); - } - - public static class Commit - { - /// A handle to an oid owned by LibGit2 - [DllImport(Git2DllName, EntryPoint = "git_commit_tree_id")] - public static extern IntPtr GetTreeId(IntPtr commitHandle); - } - - public static class Blob - { - [DllImport(Git2DllName, EntryPoint = "git_blob_rawsize")] - [return: MarshalAs(UnmanagedType.U8)] - public static extern long GetRawSize(IntPtr objectHandle); - - [DllImport(Git2DllName, EntryPoint = "git_blob_rawcontent")] - public static unsafe extern byte* GetRawContent(IntPtr objectHandle); - } - } -} \ No newline at end of file diff --git a/GVFS/GVFS.Common/Git/LibGit2Repo.cs b/GVFS/GVFS.Common/Git/LibGit2Repo.cs index aa46471e40..2536a1bab0 100644 --- a/GVFS/GVFS.Common/Git/LibGit2Repo.cs +++ b/GVFS/GVFS.Common/Git/LibGit2Repo.cs @@ -1,4 +1,5 @@ using GVFS.Common.Tracing; +using Microsoft.Diagnostics.Tracing; using Microsoft.Win32.SafeHandles; using System; using System.Collections.Generic; @@ -6,11 +7,14 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; +using System.Security.AccessControl; namespace GVFS.Common.Git { public class LibGit2Repo : IDisposable { + private const int AccessDeniedWin32Error = 5; + private ITracer tracer; private IntPtr repoHandle; private bool disposedValue = false; @@ -18,15 +22,15 @@ public class LibGit2Repo : IDisposable public LibGit2Repo(ITracer tracer, string repoPath) { this.tracer = tracer; - LibGit2Helpers.Init(); + Native.Init(); - if (LibGit2Helpers.Repo.Open(out this.repoHandle, repoPath) != LibGit2Helpers.SuccessCode) + if (Native.Repo.Open(out this.repoHandle, repoPath) != Native.SuccessCode) { - string reason = LibGit2Helpers.GetLastError(); + string reason = Native.GetLastError(); string message = "Couldn't open repo at " + repoPath + ": " + reason; tracer.RelatedError(message); - LibGit2Helpers.Shutdown(); + Native.Shutdown(); throw new InvalidDataException(message); } } @@ -39,12 +43,12 @@ public LibGit2Repo(ITracer tracer, string repoPath) public virtual bool ObjectExists(string sha) { IntPtr objHandle; - if (LibGit2Helpers.RevParseSingle(out objHandle, this.repoHandle, sha) != LibGit2Helpers.SuccessCode) + if (Native.RevParseSingle(out objHandle, this.repoHandle, sha) != Native.SuccessCode) { return false; } - LibGit2Helpers.Object.Free(objHandle); + Native.Object.Free(objHandle); return true; } @@ -53,23 +57,23 @@ public virtual bool TryGetObjectSize(string sha, out long size) size = -1; IntPtr objHandle; - if (LibGit2Helpers.RevParseSingle(out objHandle, this.repoHandle, sha) != LibGit2Helpers.SuccessCode) + if (Native.RevParseSingle(out objHandle, this.repoHandle, sha) != Native.SuccessCode) { return false; } try { - switch (LibGit2Helpers.Object.GetType(objHandle)) + switch (Native.Object.GetType(objHandle)) { - case LibGit2Helpers.ObjectTypes.Blob: - size = LibGit2Helpers.Blob.GetRawSize(objHandle); + case Native.ObjectTypes.Blob: + size = Native.Blob.GetRawSize(objHandle); return true; } } finally { - LibGit2Helpers.Object.Free(objHandle); + Native.Object.Free(objHandle); } return false; @@ -78,23 +82,23 @@ public virtual bool TryGetObjectSize(string sha, out long size) public virtual string GetTreeSha(string commitish) { IntPtr objHandle; - if (LibGit2Helpers.RevParseSingle(out objHandle, this.repoHandle, commitish) != LibGit2Helpers.SuccessCode) + if (Native.RevParseSingle(out objHandle, this.repoHandle, commitish) != Native.SuccessCode) { return null; } try { - switch (LibGit2Helpers.Object.GetType(objHandle)) + switch (Native.Object.GetType(objHandle)) { - case LibGit2Helpers.ObjectTypes.Commit: - GitOid output = LibGit2Helpers.IntPtrToGitOid(LibGit2Helpers.Commit.GetTreeId(objHandle)); + case Native.ObjectTypes.Commit: + GitOid output = Native.IntPtrToGitOid(Native.Commit.GetTreeId(objHandle)); return output.ToString(); } } finally { - LibGit2Helpers.Object.Free(objHandle); + Native.Object.Free(objHandle); } return null; @@ -103,7 +107,7 @@ public virtual string GetTreeSha(string commitish) public virtual bool TryCopyBlob(string sha, Action writeAction) { IntPtr objHandle; - if (LibGit2Helpers.RevParseSingle(out objHandle, this.repoHandle, sha) != LibGit2Helpers.SuccessCode) + if (Native.RevParseSingle(out objHandle, this.repoHandle, sha) != Native.SuccessCode) { return false; } @@ -112,14 +116,14 @@ public virtual bool TryCopyBlob(string sha, Action writeAction) { unsafe { - switch (LibGit2Helpers.Object.GetType(objHandle)) + switch (Native.Object.GetType(objHandle)) { - case LibGit2Helpers.ObjectTypes.Blob: - byte* originalData = LibGit2Helpers.Blob.GetRawContent(objHandle); - long originalSize = LibGit2Helpers.Blob.GetRawSize(objHandle); + case Native.ObjectTypes.Blob: + byte* originalData = Native.Blob.GetRawContent(objHandle); + long originalSize = Native.Blob.GetRawSize(objHandle); // TODO 938696: UnmanagedMemoryStream marshals content even for CopyTo - // If GetRawContent changed to return IntPtr and GvfltWrapper changed GVFltWriteBuffer to expose an IntPtr, + // If GetRawContent changed to return IntPtr and GvFlt changed WriteBuffer to expose an IntPtr, // We could probably pinvoke memcpy and avoid marshalling. using (Stream mem = new UnmanagedMemoryStream(originalData, originalSize)) { @@ -134,7 +138,7 @@ public virtual bool TryCopyBlob(string sha, Action writeAction) } finally { - LibGit2Helpers.Object.Free(objHandle); + Native.Object.Free(objHandle); } return true; @@ -143,7 +147,7 @@ public virtual bool TryCopyBlob(string sha, Action writeAction) public virtual bool TryCopyBlobToFile(string sha, IEnumerable destinations, out long bytesWritten) { IntPtr objHandle; - if (LibGit2Helpers.RevParseSingle(out objHandle, this.repoHandle, sha) != LibGit2Helpers.SuccessCode) + if (Native.RevParseSingle(out objHandle, this.repoHandle, sha) != Native.SuccessCode) { bytesWritten = 0; EventMetadata metadata = new EventMetadata(); @@ -158,17 +162,17 @@ public virtual bool TryCopyBlobToFile(string sha, IEnumerable destinatio // Avoid marshalling raw content by using byte* and native writes unsafe { - switch (LibGit2Helpers.Object.GetType(objHandle)) + switch (Native.Object.GetType(objHandle)) { - case LibGit2Helpers.ObjectTypes.Blob: - byte* originalData = LibGit2Helpers.Blob.GetRawContent(objHandle); - long originalSize = LibGit2Helpers.Blob.GetRawSize(objHandle); + case Native.ObjectTypes.Blob: + byte* originalData = Native.Blob.GetRawContent(objHandle); + long originalSize = Native.Blob.GetRawSize(objHandle); foreach (string destination in destinations) { try { - using (SafeFileHandle fileHandle = OpenForWrite(destination)) + using (SafeFileHandle fileHandle = OpenForWrite(this.tracer, destination)) { if (fileHandle.IsInvalid) { @@ -181,7 +185,7 @@ public virtual bool TryCopyBlobToFile(string sha, IEnumerable destinatio while (size > 0) { uint toWrite = size < uint.MaxValue ? (uint)size : uint.MaxValue; - if (!WriteFile(fileHandle, data, toWrite, out written, IntPtr.Zero)) + if (!Native.WriteFile(fileHandle, data, toWrite, out written, IntPtr.Zero)) { throw new Win32Exception(Marshal.GetLastWin32Error()); } @@ -194,7 +198,7 @@ public virtual bool TryCopyBlobToFile(string sha, IEnumerable destinatio catch (Exception e) { this.tracer.RelatedError("Exception writing {0}: {1}", destination, e); - throw e; + throw; } } @@ -207,7 +211,7 @@ public virtual bool TryCopyBlobToFile(string sha, IEnumerable destinatio } finally { - LibGit2Helpers.Object.Free(objHandle); + Native.Object.Free(objHandle); } return true; @@ -223,29 +227,141 @@ protected virtual void Dispose(bool disposing) { if (!this.disposedValue) { - LibGit2Helpers.Repo.Free(this.repoHandle); - LibGit2Helpers.Shutdown(); + Native.Repo.Free(this.repoHandle); + Native.Shutdown(); this.disposedValue = true; } } - private static SafeFileHandle OpenForWrite(string fileName) + private static SafeFileHandle OpenForWrite(ITracer tracer, string fileName) { - return CreateFile(fileName, FileAccess.Write, FileShare.None, IntPtr.Zero, FileMode.Create, FileAttributes.Normal, IntPtr.Zero); + SafeFileHandle handle = Native.CreateFile(fileName, FileAccess.Write, FileShare.None, IntPtr.Zero, FileMode.Create, FileAttributes.Normal, IntPtr.Zero); + if (handle.IsInvalid) + { + // If we get a access denied, try reverting the acls to defaults inherited by parent + if (Marshal.GetLastWin32Error() == AccessDeniedWin32Error) + { + tracer.RelatedEvent( + EventLevel.Warning, + "FailedOpenForWrite", + new EventMetadata + { + { "WarningMessage", "Received access denied. Resetting ACLs to default." }, + { "FileName", fileName } + }); + + FileSecurity fs = new FileSecurity(); + fs.SetAccessRuleProtection(false, false); + File.SetAccessControl(fileName, fs); + + handle = Native.CreateFile(fileName, FileAccess.Write, FileShare.None, IntPtr.Zero, FileMode.Create, FileAttributes.Normal, IntPtr.Zero); + } + } + + return handle; } - [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] - private static extern SafeFileHandle CreateFile( - [MarshalAs(UnmanagedType.LPTStr)] string filename, - [MarshalAs(UnmanagedType.U4)] FileAccess access, - [MarshalAs(UnmanagedType.U4)] FileShare share, - IntPtr securityAttributes, // optional SECURITY_ATTRIBUTES struct or IntPtr.Zero - [MarshalAs(UnmanagedType.U4)] FileMode creationDisposition, - [MarshalAs(UnmanagedType.U4)] FileAttributes flagsAndAttributes, - IntPtr templateFile); - - [DllImport("kernel32.dll")] - [return: MarshalAs(UnmanagedType.Bool)] - private static unsafe extern bool WriteFile(SafeFileHandle file, byte* buffer, uint numberOfBytesToWrite, out uint numberOfBytesWritten, IntPtr overlapped); + public static class Native + { + public const uint SuccessCode = 0; + + public const string Git2DllName = "git2.dll"; + + public enum ObjectTypes + { + Commit = 1, + Tree = 2, + Blob = 3, + } + + public static GitOid IntPtrToGitOid(IntPtr oidPtr) + { + return Marshal.PtrToStructure(oidPtr); + } + + [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] + public static extern SafeFileHandle CreateFile( + [MarshalAs(UnmanagedType.LPTStr)] string filename, + [MarshalAs(UnmanagedType.U4)] FileAccess access, + [MarshalAs(UnmanagedType.U4)] FileShare share, + IntPtr securityAttributes, // optional SECURITY_ATTRIBUTES struct or IntPtr.Zero + [MarshalAs(UnmanagedType.U4)] FileMode creationDisposition, + [MarshalAs(UnmanagedType.U4)] FileAttributes flagsAndAttributes, + IntPtr templateFile); + + [DllImport("kernel32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + public static unsafe extern bool WriteFile(SafeFileHandle file, byte* buffer, uint numberOfBytesToWrite, out uint numberOfBytesWritten, IntPtr overlapped); + + [DllImport(Git2DllName, EntryPoint = "git_libgit2_init")] + public static extern void Init(); + + [DllImport(Git2DllName, EntryPoint = "git_libgit2_shutdown")] + public static extern int Shutdown(); + + [DllImport(Git2DllName, EntryPoint = "git_revparse_single")] + public static extern uint RevParseSingle(out IntPtr objectHandle, IntPtr repoHandle, string oid); + + [DllImport(Git2DllName, EntryPoint = "git_oid_fromstr")] + public static extern void OidFromString(ref GitOid oid, string hash); + + public static string GetLastError() + { + IntPtr ptr = GetLastGitError(); + if (ptr == IntPtr.Zero) + { + return "Operation was successful"; + } + + return Marshal.PtrToStructure(ptr).Message; + } + + [DllImport(Git2DllName, EntryPoint = "giterr_last")] + private static extern IntPtr GetLastGitError(); + + [StructLayout(LayoutKind.Sequential)] + private struct GitError + { + [MarshalAs(UnmanagedType.LPStr)] + public string Message; + + public int Klass; + } + + public static class Repo + { + [DllImport(Git2DllName, EntryPoint = "git_repository_open")] + public static extern uint Open(out IntPtr repoHandle, string path); + + [DllImport(Git2DllName, EntryPoint = "git_tree_free")] + public static extern void Free(IntPtr repoHandle); + } + + public static class Object + { + [DllImport(Git2DllName, EntryPoint = "git_object_type")] + public static extern ObjectTypes GetType(IntPtr objectHandle); + + [DllImport(Git2DllName, EntryPoint = "git_object_free")] + public static extern void Free(IntPtr objHandle); + } + + public static class Commit + { + /// A handle to an oid owned by LibGit2 + [DllImport(Git2DllName, EntryPoint = "git_commit_tree_id")] + public static extern IntPtr GetTreeId(IntPtr commitHandle); + } + + public static class Blob + { + [DllImport(Git2DllName, EntryPoint = "git_blob_rawsize")] + [return: MarshalAs(UnmanagedType.U8)] + public static extern long GetRawSize(IntPtr objectHandle); + + [DllImport(Git2DllName, EntryPoint = "git_blob_rawcontent")] + public static unsafe extern byte* GetRawContent(IntPtr objectHandle); + } + } } } \ No newline at end of file diff --git a/GVFS/GVFS.Common/Git/RefLogEntry.cs b/GVFS/GVFS.Common/Git/RefLogEntry.cs index ad8f5ec0c2..1c242dab6e 100644 --- a/GVFS/GVFS.Common/Git/RefLogEntry.cs +++ b/GVFS/GVFS.Common/Git/RefLogEntry.cs @@ -1,6 +1,4 @@ -using System; - -namespace GVFS.Common.Git +namespace GVFS.Common.Git { public class RefLogEntry { @@ -18,6 +16,11 @@ public RefLogEntry(string sourceSha, string targetSha, string reason) public static bool TryParse(string line, out RefLogEntry entry) { entry = null; + if (string.IsNullOrEmpty(line)) + { + return false; + } + if (line.Length < GVFSConstants.ShaStringLength + 1 + GVFSConstants.ShaStringLength) { return false; diff --git a/GVFS/GVFS.Common/GitCommandLineParser.cs b/GVFS/GVFS.Common/GitCommandLineParser.cs new file mode 100644 index 0000000000..725d65e428 --- /dev/null +++ b/GVFS/GVFS.Common/GitCommandLineParser.cs @@ -0,0 +1,135 @@ +using System; +using System.Linq; + +namespace GVFS.Common +{ + public class GitCommandLineParser + { + private const int GitIndex = 0; + private const int VerbIndex = 1; + private const int ArgumentsOffset = 2; + + private readonly string[] parts; + + public GitCommandLineParser(string command) + { + if (!string.IsNullOrWhiteSpace(command)) + { + this.parts = command.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + + if (this.parts.Length < VerbIndex + 1 || + this.parts[GitIndex] != "git") + { + this.parts = null; + } + } + } + + public bool IsValidGitCommand + { + get { return this.parts != null; } + } + + public bool IsResetSoftOrMixed() + { + return + this.IsVerb("reset") && + !this.HasArgument("--hard") && + !this.HasArgument("--keep") && + !this.HasArgument("--merge"); + } + + public bool IsResetHard() + { + return this.IsVerb("reset") && this.HasArgument("--hard"); + } + + /// + /// This method currently just makes a best effort to detect file paths. Only use this method for optional optimizations + /// related to file paths. Do NOT use this method if you require a reliable answer. + /// + /// True if file paths were detected, otherwise false + public bool IsCheckoutWithFilePaths() + { + if (this.IsVerb("checkout")) + { + int numArguments = this.parts.Length - ArgumentsOffset; + + // The simplest way to know that we're dealing with file paths is if there are any arguments after a -- + // e.g. git checkout branchName -- fileName + int dashDashIndex; + if (this.HasAnyArgument(arg => arg == "--", out dashDashIndex) && + numArguments > dashDashIndex + 1) + { + return true; + } + + // We also special case one usage with HEAD, as long as there are no other arguments with - or -- that might + // result in behavior we haven't tested. + // e.g. git checkout HEAD fileName + if (numArguments >= 2 && + !this.HasAnyArgument(arg => arg.StartsWith("-")) && + this.HasArgumentAtIndex(GVFSConstants.DotGit.HeadName, argumentIndex: 0)) + { + return true; + } + + // Note: we have definitely missed some cases of file paths, e.g.: + // git checkout branchName fileName (detecting this reliably requires more complicated parsing) + // git checkout --patch (we currently have no need to optimize this scenario) + } + + return false; + } + + public bool IsVerb(params string[] verbs) + { + if (!this.IsValidGitCommand) + { + return false; + } + + return verbs.Any(v => this.parts[VerbIndex] == v); + } + + private bool HasArgument(string argument) + { + return this.HasAnyArgument(arg => arg == argument); + } + + private bool HasArgumentAtIndex(string argument, int argumentIndex) + { + int actualIndex = argumentIndex + ArgumentsOffset; + return + this.parts.Length > actualIndex && + this.parts[actualIndex] == argument; + } + + private bool HasAnyArgument(Predicate argumentPredicate) + { + int argumentIndex; + return this.HasAnyArgument(argumentPredicate, out argumentIndex); + } + + private bool HasAnyArgument(Predicate argumentPredicate, out int argumentIndex) + { + argumentIndex = -1; + + if (!this.IsValidGitCommand) + { + return false; + } + + for (int i = ArgumentsOffset; i < this.parts.Length; i++) + { + if (argumentPredicate(this.parts[i])) + { + argumentIndex = i - ArgumentsOffset; + return true; + } + } + + return false; + } + } +} diff --git a/GVFS/GVFS.Common/GitHelper.cs b/GVFS/GVFS.Common/GitHelper.cs deleted file mode 100644 index 1483067b16..0000000000 --- a/GVFS/GVFS.Common/GitHelper.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Linq; - -namespace GVFS.Common -{ - public static class GitHelper - { - /// - /// Determines whether the given command line represents any of the git verbs passed in. - /// - /// The git command line. - /// A list of verbs (eg. "status" not "git status"). - /// True if the command line represents any of the verbs, false otherwise. - public static bool IsVerb(string commandLine, params string[] verbs) - { - if (verbs == null || verbs.Length < 1) - { - throw new ArgumentException("At least one verb must be provided.", nameof(verbs)); - } - - return - verbs.Any(v => - { - string verbCommand = "git " + v; - return - commandLine == verbCommand || - commandLine.StartsWith(verbCommand + " "); - }); - } - - /// - /// Returns true if the string is length 40 and all valid hex characters - /// - public static bool IsValidFullSHA(string sha) - { - return sha.Length == 40 && !sha.Any(c => !(c >= '0' && c <= '9') && !(c >= 'a' && c <= 'f') && !(c >= 'A' && c <= 'F')); - } - } -} diff --git a/GVFS/GVFS.Common/Http/CacheServerInfo.cs b/GVFS/GVFS.Common/Http/CacheServerInfo.cs new file mode 100644 index 0000000000..d6cd5a747a --- /dev/null +++ b/GVFS/GVFS.Common/Http/CacheServerInfo.cs @@ -0,0 +1,248 @@ +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace GVFS.Common.Http +{ + public class CacheServerInfo + { + public const string NoneFriendlyName = "None"; + public const string UserDefinedFriendlyName = "User Defined"; + + private const string CacheServerConfigName = "gvfs.cache-server"; + private const string ObjectsEndpointSuffix = "/gvfs/objects"; + private const string PrefetchEndpointSuffix = "/gvfs/prefetch"; + + private const string DeprecatedCacheEndpointGitConfigSuffix = ".cache-server-url"; + + [JsonConstructor] + public CacheServerInfo(string url, string name, bool globalDefault = false) + { + this.Url = url; + this.Name = name; + this.GlobalDefault = globalDefault; + + this.ObjectsEndpointUrl = this.Url + ObjectsEndpointSuffix; + this.PrefetchEndpointUrl = this.Url + PrefetchEndpointSuffix; + } + + public string Url { get; } + public string Name { get; } + public bool GlobalDefault { get; } + + public string ObjectsEndpointUrl { get; } + public string PrefetchEndpointUrl { get; } + + public static bool TryDetermineCacheServer( + string userUrlish, + ITracer tracer, + Enlistment enlistment, + RetryConfig retryConfig, + out CacheServerInfo cache, + out string error) + { + using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) + { + GVFSConfig config = configRequestor.QueryGVFSConfig(); + if (!CacheServerInfo.TryDetermineCacheServer(userUrlish, enlistment, config.CacheServers, out cache, out error)) + { + return false; + } + } + + return true; + } + + public static bool TryDetermineCacheServer( + string userUrlish, + Enlistment enlistment, + IEnumerable knownCaches, + out CacheServerInfo output, + out string error) + { + return TryDetermineCacheServer(userUrlish, new GitProcess(enlistment), enlistment, knownCaches, out output, out error); + } + + public static bool TryDetermineCacheServer( + string userUrlish, + GitProcess gitProcess, + Enlistment enlistment, + IEnumerable knownCaches, + out CacheServerInfo output, + out string error) + { + output = null; + + // User input overrules everything + if (!string.IsNullOrWhiteSpace(userUrlish)) + { + if (!CacheServerInfo.TryParse(userUrlish, enlistment, knownCaches, out output)) + { + error = "Failed to determine remote objects endpoint"; + return false; + } + + error = null; + return true; + } + + // Fallback to git config entry if possible. + string configCacheServer = GetValueFromConfig(gitProcess, CacheServerConfigName); + if (!string.IsNullOrWhiteSpace(configCacheServer)) + { + if (!CacheServerInfo.TryParse(configCacheServer, enlistment, knownCaches, out output)) + { + error = string.Format( + "Failed to determine remote objects endpoint from '{0}': {1}", + CacheServerConfigName, + configCacheServer); + + error = "Failed to determine remote objects endpoint"; + return false; + } + + error = null; + return true; + } + + // Fallback to deprecated cache git config entry (as upgrade path) + // TODO 1057500: Someday remove support for encoded-repo-url cache config setting + string deprecatedConfigSetting = GetDeprecatedCacheConfigSettingName(enlistment.RepoUrl); + string deprecatedConfigCacheServer = GetValueFromConfig(gitProcess, deprecatedConfigSetting); + if (!string.IsNullOrWhiteSpace(deprecatedConfigCacheServer)) + { + if (!CacheServerInfo.TryParse(deprecatedConfigCacheServer, enlistment, knownCaches, out output)) + { + error = string.Format( + "Failed to determine remote objects endpoint from '{0}': {1}", + deprecatedConfigSetting, + deprecatedConfigCacheServer); + + return false; + } + + if (!TrySaveToConfig(gitProcess, output, out error)) + { + return false; + } + + error = null; + return true; + } + + // Fallback to any known default cache. + if (knownCaches != null) + { + output = knownCaches.FirstOrDefault(cache => cache.GlobalDefault); + } + + // If there are no known defaults, then default to None + if (output == null) + { + output = new CacheServerInfo(enlistment.RepoUrl, NoneFriendlyName); + } + + error = null; + return true; + } + + public static bool TryParse(string urlish, Enlistment enlistment, IEnumerable knownCaches, out CacheServerInfo output) + { + output = null; + if (string.IsNullOrWhiteSpace(urlish)) + { + return false; + } + + if (urlish.Equals(NoneFriendlyName, StringComparison.OrdinalIgnoreCase) || + urlish.Equals(enlistment.RepoUrl, StringComparison.OrdinalIgnoreCase)) + { + output = new CacheServerInfo(enlistment.RepoUrl, NoneFriendlyName); + return true; + } + + if (knownCaches != null) + { + output = knownCaches.FirstOrDefault(cache => cache.Url.Equals(urlish, StringComparison.OrdinalIgnoreCase)); + if (output != null) + { + return true; + } + } + + Uri uri; + if (Uri.TryCreate(urlish, UriKind.Absolute, out uri)) + { + output = new CacheServerInfo(urlish, UserDefinedFriendlyName); + return true; + } + + if (knownCaches != null) + { + output = knownCaches.FirstOrDefault(cache => cache.Name.Equals(urlish, StringComparison.OrdinalIgnoreCase)); + + return output != null; + } + + return false; + } + + public static bool TrySaveToConfig(GitProcess git, CacheServerInfo cache, out string error) + { + string value = + cache.Name.Equals(UserDefinedFriendlyName, StringComparison.OrdinalIgnoreCase) + ? cache.Url + : cache.Name; + + GitProcess.Result result = git.SetInLocalConfig(CacheServerConfigName, value); + + error = result.Errors; + return !result.HasErrors; + } + + public static string GetCacheServerValueFromConfig(Enlistment enlistment) + { + GitProcess git = new GitProcess(enlistment); + + // TODO 1057500: Someday remove support for encoded-repo-url cache config setting + return GetValueFromConfig(git, CacheServerConfigName) + ?? GetValueFromConfig(git, GetDeprecatedCacheConfigSettingName(enlistment.RepoUrl)); + } + + public override string ToString() + { + return string.Format("{0} ({1})", this.Name, this.Url); + } + + private static string GetValueFromConfig(GitProcess git, string configName) + { + GitProcess.Result result = git.GetFromConfig(configName); + + // Git returns non-zero for non-existent settings and errors. + if (!result.HasErrors) + { + return result.Output.TrimEnd('\n'); + } + else if (result.Errors.Any()) + { + throw new InvalidRepoException("Error while reading '" + configName + "' from config: " + result.Errors); + } + + return null; + } + + private static string GetDeprecatedCacheConfigSettingName(string repoUrl) + { + string sectionUrl = + repoUrl.ToLowerInvariant() + .Replace("https://", string.Empty) + .Replace("http://", string.Empty) + .Replace('/', '.'); + + return GVFSConstants.GitConfig.GVFSPrefix + sectionUrl + DeprecatedCacheEndpointGitConfigSuffix; + } + } +} diff --git a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs index 8252fdf4d8..fab52871d8 100644 --- a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs @@ -1,10 +1,9 @@ -using GVFS.Common.Physical.FileSystem; +using GVFS.Common.NetworkStreams; using GVFS.Common.Tracing; using Newtonsoft.Json; using System; using System.IO; using System.Net.Http; -using System.Threading; namespace GVFS.Common.Http { @@ -12,15 +11,12 @@ public class ConfigHttpRequestor : HttpRequestor { private readonly string repoUrl; - public ConfigHttpRequestor(ITracer tracer, Enlistment enlistment) - : base(tracer, enlistment.Authentication) + public ConfigHttpRequestor(ITracer tracer, Enlistment enlistment, RetryConfig retryConfig) + : base(tracer, retryConfig, enlistment.Authentication) { this.repoUrl = enlistment.RepoUrl; - this.MaxRetries = HttpRequestor.DefaultMaxRetries; } - public int MaxRetries { get; set; } - public GVFSConfig QueryGVFSConfig() { Uri gvfsConfigEndpoint; @@ -41,7 +37,7 @@ public GVFSConfig QueryGVFSConfig() } long requestId = HttpRequestor.GetNewRequestId(); - RetryWrapper retrier = new RetryWrapper(this.MaxRetries); + RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts); retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); RetryWrapper.InvocationResult output = retrier.Invoke( @@ -58,8 +54,8 @@ public GVFSConfig QueryGVFSConfig() using (StreamReader reader = new StreamReader(response.Stream)) { string configString = reader.RetryableReadToEnd(); - return new RetryWrapper.CallbackResult( - JsonConvert.DeserializeObject(configString)); + GVFSConfig config = JsonConvert.DeserializeObject(configString); + return new RetryWrapper.CallbackResult(config); } } catch (JsonReaderException e) diff --git a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs index 6c59c053b3..a8a9e56b72 100644 --- a/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/GitObjectsHttpRequestor.cs @@ -1,5 +1,5 @@ using GVFS.Common.Git; -using GVFS.Common.Physical.FileSystem; +using GVFS.Common.NetworkStreams; using GVFS.Common.Tracing; using Microsoft.Diagnostics.Tracing; using Newtonsoft.Json; @@ -10,25 +10,26 @@ using System.Net; using System.Net.Http; using System.Net.Http.Headers; -using System.Threading; namespace GVFS.Common.Http { public class GitObjectsHttpRequestor : HttpRequestor { + public const int UseConfiguredMaxAttempts = -1; + private static readonly MediaTypeWithQualityHeaderValue CustomLooseObjectsHeader = new MediaTypeWithQualityHeaderValue(GVFSConstants.MediaTypes.CustomLooseObjectsMediaType); - + private Enlistment enlistment; - - public GitObjectsHttpRequestor(ITracer tracer, Enlistment enlistment) - : base(tracer, enlistment.Authentication) + + public GitObjectsHttpRequestor(ITracer tracer, Enlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig) + : base(tracer, retryConfig, enlistment.Authentication) { this.enlistment = enlistment; - this.MaxRetries = HttpRequestor.DefaultMaxRetries; + this.CacheServer = cacheServer; } - - public int MaxRetries { get; set; } + + public CacheServerInfo CacheServer { get; private set; } public virtual List QueryForFileSizes(IEnumerable objectIds) { @@ -51,7 +52,7 @@ public virtual List QueryForFileSizes(IEnumerable objectI this.Tracer.RelatedEvent(EventLevel.Informational, "QueryFileSizes", metadata, Keywords.Network); - RetryWrapper> retrier = new RetryWrapper>(this.MaxRetries); + RetryWrapper> retrier = new RetryWrapper>(this.RetryConfig.MaxAttempts); retrier.OnFailure += RetryWrapper>.StandardErrorHandler(this.Tracer, requestId, "QueryFileSizes"); RetryWrapper>.InvocationResult requestTask = retrier.Invoke( @@ -88,7 +89,7 @@ public virtual GitRefs QueryInfoRefs(string branch) return null; } - RetryWrapper retrier = new RetryWrapper(this.MaxRetries); + RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts); retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryInfoRefs"); RetryWrapper.InvocationResult output = retrier.Invoke( @@ -113,10 +114,19 @@ public virtual GitRefs QueryInfoRefs(string branch) public virtual RetryWrapper.InvocationResult TryDownloadLooseObject( string objectId, Func.CallbackResult> onSuccess) + { + return this.TryDownloadLooseObject(objectId, UseConfiguredMaxAttempts, onSuccess: onSuccess); + } + + public virtual RetryWrapper.InvocationResult TryDownloadLooseObject( + string objectId, + int maxAttempts, + Func.CallbackResult> onSuccess) { long requestId = HttpRequestor.GetNewRequestId(); EventMetadata metadata = new EventMetadata(); metadata.Add("ObjectId", objectId); + metadata.Add("maxAttempts", maxAttempts == UseConfiguredMaxAttempts ? "UseConfiguredMaxAttempts" : maxAttempts.ToString()); metadata.Add("RequestId", requestId); this.Tracer.RelatedEvent(EventLevel.Informational, "DownloadLooseObject", metadata, Keywords.Network); @@ -125,7 +135,10 @@ public virtual GitRefs QueryInfoRefs(string branch) onSuccess, eArgs => this.HandleDownloadAndSaveObjectError(requestId, eArgs), HttpMethod.Get, - new Uri(this.enlistment.ObjectsEndpointUrl + "/" + objectId)); + new Uri(this.CacheServer.ObjectsEndpointUrl + "/" + objectId), + requestBody: null, + acceptType: null, + maxAttempts: maxAttempts); } public virtual RetryWrapper.InvocationResult TryDownloadObjects( @@ -143,7 +156,7 @@ public virtual GitRefs QueryInfoRefs(string branch) onSuccess, onFailure, HttpMethod.Post, - new Uri(this.enlistment.ObjectsEndpointUrl), + new Uri(this.CacheServer.ObjectsEndpointUrl), () => this.ObjectIdsJsonGenerator(requestId, objectIdGenerator, commitDepth), preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); } @@ -177,7 +190,7 @@ public virtual GitRefs QueryInfoRefs(string branch) onSuccess, onFailure, HttpMethod.Post, - new Uri(this.enlistment.ObjectsEndpointUrl), + new Uri(this.CacheServer.ObjectsEndpointUrl), objectIdsJson, preferBatchedLooseObjects ? CustomLooseObjectsHeader : null); } @@ -189,7 +202,8 @@ public virtual GitRefs QueryInfoRefs(string branch) HttpMethod method, Uri endPoint, string requestBody = null, - MediaTypeWithQualityHeaderValue acceptType = null) + MediaTypeWithQualityHeaderValue acceptType = null, + int maxAttempts = UseConfiguredMaxAttempts) { return this.TrySendProtocolRequest( requestId, @@ -198,7 +212,8 @@ public virtual GitRefs QueryInfoRefs(string branch) method, endPoint, () => requestBody, - acceptType); + acceptType, + maxAttempts); } public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( @@ -208,7 +223,8 @@ public virtual GitRefs QueryInfoRefs(string branch) HttpMethod method, Uri endPoint, Func requestBodyGenerator, - MediaTypeWithQualityHeaderValue acceptType = null) + MediaTypeWithQualityHeaderValue acceptType = null, + int maxAttempts = UseConfiguredMaxAttempts) { return this.TrySendProtocolRequest( requestId, @@ -217,7 +233,8 @@ public virtual GitRefs QueryInfoRefs(string branch) method, () => endPoint, requestBodyGenerator, - acceptType); + acceptType, + maxAttempts); } public virtual RetryWrapper.InvocationResult TrySendProtocolRequest( @@ -227,9 +244,10 @@ public virtual GitRefs QueryInfoRefs(string branch) HttpMethod method, Func endPointGenerator, Func requestBodyGenerator, - MediaTypeWithQualityHeaderValue acceptType = null) + MediaTypeWithQualityHeaderValue acceptType = null, + int maxAttempts = UseConfiguredMaxAttempts) { - RetryWrapper retrier = new RetryWrapper(this.MaxRetries); + RetryWrapper retrier = new RetryWrapper(maxAttempts == UseConfiguredMaxAttempts ? this.RetryConfig.MaxAttempts : maxAttempts); if (onFailure != null) { retrier.OnFailure += onFailure; diff --git a/GVFS/GVFS.Common/Http/HttpRequestor.cs b/GVFS/GVFS.Common/Http/HttpRequestor.cs index de0331eea0..1862f217c1 100644 --- a/GVFS/GVFS.Common/Http/HttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/HttpRequestor.cs @@ -16,13 +16,10 @@ namespace GVFS.Common.Http { public abstract class HttpRequestor : IDisposable { - public const int DefaultMaxRetries = 5; - private const int HttpTimeoutMinutes = 10; - private static long requestCount = 0; private readonly ProductInfoHeaderValue userAgentHeader; - + private HttpClient client; private GitAuthentication authentication; @@ -31,17 +28,20 @@ static HttpRequestor() ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount; } - public HttpRequestor(ITracer tracer, GitAuthentication authentication) + public HttpRequestor(ITracer tracer, RetryConfig retryConfig, GitAuthentication authentication) { - this.client = new HttpClient(); - this.client.Timeout = TimeSpan.FromMinutes(HttpTimeoutMinutes); + this.client = new HttpClient(new HttpClientHandler() { UseDefaultCredentials = true }); + this.client.Timeout = retryConfig.Timeout; + this.RetryConfig = retryConfig; this.authentication = authentication; this.Tracer = tracer; - + this.userAgentHeader = new ProductInfoHeaderValue(ProcessHelper.GetEntryClassName(), ProcessHelper.GetCurrentProcessVersion()); } + public RetryConfig RetryConfig { get; } + protected ITracer Tracer { get; } public static long GetNewRequestId() @@ -72,12 +72,16 @@ public void Dispose() return new GitEndPointResponseData( HttpStatusCode.Unauthorized, new GitObjectsHttpException(HttpStatusCode.Unauthorized, errorMessage), - shouldRetry: false); + shouldRetry: true); } HttpRequestMessage request = new HttpRequestMessage(httpMethod, requestUri); request.Headers.UserAgent.Add(this.userAgentHeader); - request.Headers.Authorization = new AuthenticationHeaderValue("Basic", authString); + + if (!string.IsNullOrEmpty(authString)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", authString); + } if (acceptType != null) { @@ -117,19 +121,14 @@ public void Dispose() { if (response.StatusCode == HttpStatusCode.Unauthorized) { - if (this.authentication.RevokeAndCheckCanRetry(authString)) + this.authentication.Revoke(authString); + if (!this.authentication.IsBackingOff) { - return new GitEndPointResponseData( - response.StatusCode, - new GitObjectsHttpException(response.StatusCode, "Server returned error code 401 (Unauthorized). Your PAT may be expired."), - shouldRetry: true); + errorMessage = "Server returned error code 401 (Unauthorized). Your PAT may be expired and we are asking for a new one."; } else { - return new GitEndPointResponseData( - response.StatusCode, - new GitObjectsHttpException(response.StatusCode, "Server returned error code 401 (Unauthorized) after successfully renewing your PAT. You may not have access to this repo"), - shouldRetry: false); + errorMessage = "Server returned error code 401 (Unauthorized) after successfully renewing your PAT. You may not have access to this repo"; } } else @@ -158,9 +157,10 @@ public void Dispose() private static bool ShouldRetry(HttpStatusCode statusCode) { - // Retry timeouts and 5xx errors + // Retry timeout, Unauthorized, and 5xx errors int statusInt = (int)statusCode; if (statusCode == HttpStatusCode.RequestTimeout || + statusCode == HttpStatusCode.Unauthorized || (statusInt >= 500 && statusInt < 600)) { return true; diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs index 42b8787c75..e2a5a2905d 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs @@ -18,7 +18,7 @@ public NamedPipeClient(string pipeName) public static string GetPipeNameFromPath(string path) { - return "GVFS_" + path.ToUpper().Replace(':', '_'); + return Paths.GetNamedPipeName(path); } public bool Connect(int timeoutMilliseconds = 3000) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeMessages.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeMessages.cs index 63333d74c3..3de8318023 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeMessages.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeMessages.cs @@ -39,7 +39,7 @@ public class Response public string MountStatus { get; set; } public string EnlistmentRoot { get; set; } public string RepoUrl { get; set; } - public string ObjectsUrl { get; set; } + public string CacheServer { get; set; } public int BackgroundOperationCount { get; set; } public string LockStatus { get; set; } public int DiskLayoutVersion { get; set; } @@ -127,15 +127,15 @@ public Message ToMessage() } } - public class UnmountRepoRequest + public class UnregisterRepoRequest { - public const string Header = nameof(UnmountRepoRequest); + public const string Header = nameof(UnregisterRepoRequest); public string EnlistmentRoot { get; set; } - public static UnmountRepoRequest FromMessage(Message message) + public static UnregisterRepoRequest FromMessage(Message message) { - return JsonConvert.DeserializeObject(message.Body); + return JsonConvert.DeserializeObject(message.Body); } public Message ToMessage() @@ -143,38 +143,50 @@ public Message ToMessage() return new Message(Header, JsonConvert.SerializeObject(this)); } - public class Response + public class Response : BaseResponse { - public const string Header = nameof(UnmountRepoRequest) + ResponseSuffix; - - public CompletionState State { get; set; } - - public string UserText { get; set; } - public static Response FromMessage(Message message) { return JsonConvert.DeserializeObject(message.Body); } + } + } - public Message ToMessage() + public class RegisterRepoRequest + { + public const string Header = nameof(RegisterRepoRequest); + + public string EnlistmentRoot { get; set; } + public string OwnerSID { get; set; } + + public static RegisterRepoRequest FromMessage(Message message) + { + return JsonConvert.DeserializeObject(message.Body); + } + + public Message ToMessage() + { + return new Message(Header, JsonConvert.SerializeObject(this)); + } + + public class Response : BaseResponse + { + public static Response FromMessage(Message message) { - return new Message(Header, JsonConvert.SerializeObject(this)); + return JsonConvert.DeserializeObject(message.Body); } } } - public class MountRepoRequest + public class AttachGvFltRequest { - public const string Header = nameof(MountRepoRequest); + public const string Header = nameof(AttachGvFltRequest); public string EnlistmentRoot { get; set; } - public Keywords Keywords { get; set; } - public bool ShowDebugWindow { get; set; } - public EventLevel Verbosity { get; set; } - public static MountRepoRequest FromMessage(Message message) + public static AttachGvFltRequest FromMessage(Message message) { - return JsonConvert.DeserializeObject(message.Body); + return JsonConvert.DeserializeObject(message.Body); } public Message ToMessage() @@ -182,22 +194,25 @@ public Message ToMessage() return new Message(Header, JsonConvert.SerializeObject(this)); } - public class Response + public class Response : BaseResponse { - public const string Header = nameof(MountRepoRequest) + ResponseSuffix; - - public CompletionState State { get; set; } - public string ErrorMessage { get; set; } - public static Response FromMessage(Message message) { return JsonConvert.DeserializeObject(message.Body); } + } + } - public Message ToMessage() - { - return new Message(Header, JsonConvert.SerializeObject(this)); - } + public class BaseResponse + { + public const string Header = nameof(TRequest) + ResponseSuffix; + + public CompletionState State { get; set; } + public string ErrorMessage { get; set; } + + public Message ToMessage() + { + return new Message(Header, JsonConvert.SerializeObject(this)); } } } diff --git a/GVFS/GVFS.Common/BatchedLooseObjects/BatchedLooseObjectDeserializer.cs b/GVFS/GVFS.Common/NetworkStreams/BatchedLooseObjectDeserializer.cs similarity index 91% rename from GVFS/GVFS.Common/BatchedLooseObjects/BatchedLooseObjectDeserializer.cs rename to GVFS/GVFS.Common/NetworkStreams/BatchedLooseObjectDeserializer.cs index 17943336ac..373ea6e54a 100644 --- a/GVFS/GVFS.Common/BatchedLooseObjects/BatchedLooseObjectDeserializer.cs +++ b/GVFS/GVFS.Common/NetworkStreams/BatchedLooseObjectDeserializer.cs @@ -3,13 +3,8 @@ using System.Linq; using System.Text; -namespace GVFS.Common +namespace GVFS.Common.NetworkStreams { - /// - /// Invoked when the full content of a single loose object is available. - /// - public delegate void OnLooseObject(Stream objectStream, string sha1); - /// /// Deserializer for concatenated loose objects. /// @@ -33,6 +28,11 @@ public BatchedLooseObjectDeserializer(Stream source, OnLooseObject onLooseObject this.onLooseObject = onLooseObject; } + /// + /// Invoked when the full content of a single loose object is available. + /// + public delegate void OnLooseObject(Stream objectStream, string sha1); + /// /// Read all the objects from the source stream and call for each. /// @@ -57,7 +57,7 @@ public int ProcessObjects() long curLength = BitConverter.ToInt64(curObjectHeader, NumObjectIdBytes); // Handle the loose object - using (Stream rawObjectData = new RestrictedStream(this.source, 0, curLength, leaveOpen: true)) + using (Stream rawObjectData = new RestrictedStream(this.source, curLength)) { string objectId = SHA1Util.HexStringFromBytes(curObjectHeader, NumObjectIdBytes); diff --git a/GVFS/GVFS.Common/PrefetchPacks/PrefetchPacksDeserializer.cs b/GVFS/GVFS.Common/NetworkStreams/PrefetchPacksDeserializer.cs similarity index 90% rename from GVFS/GVFS.Common/PrefetchPacks/PrefetchPacksDeserializer.cs rename to GVFS/GVFS.Common/NetworkStreams/PrefetchPacksDeserializer.cs index ee26af1bf4..7112516aae 100644 --- a/GVFS/GVFS.Common/PrefetchPacks/PrefetchPacksDeserializer.cs +++ b/GVFS/GVFS.Common/NetworkStreams/PrefetchPacksDeserializer.cs @@ -4,7 +4,7 @@ using System.Linq; using System.Text; -namespace GVFS.Common +namespace GVFS.Common.NetworkStreams { /// /// Deserializer for packs and indexes for prefetch packs. @@ -13,8 +13,8 @@ public class PrefetchPacksDeserializer { private const int NumPackHeaderBytes = 3 * sizeof(long); - private static readonly byte[] PrefetchPackExpectedHeader - = new byte[] + private static readonly byte[] PrefetchPackExpectedHeader = + new byte[] { (byte)'G', (byte)'P', (byte)'R', (byte)'E', (byte)' ', // Magic 1 // Version @@ -22,8 +22,7 @@ public class PrefetchPacksDeserializer private readonly Stream source; - public PrefetchPacksDeserializer( - Stream source) + public PrefetchPacksDeserializer(Stream source) { this.source = source; } @@ -36,7 +35,6 @@ public IEnumerable EnumeratePacks() { this.ValidateHeader(); - // Start reading objects byte[] buffer = new byte[NumPackHeaderBytes]; int packCount = this.ReadPackCount(buffer); @@ -48,8 +46,8 @@ public IEnumerable EnumeratePacks() long indexLength; this.ReadPackHeader(buffer, out timestamp, out packLength, out indexLength); - using (Stream packData = new RestrictedStream(this.source, 0, packLength, leaveOpen: true)) - using (Stream indexData = indexLength > 0 ? new RestrictedStream(this.source, 0, indexLength, leaveOpen: true) : null) + using (Stream packData = new RestrictedStream(this.source, packLength)) + using (Stream indexData = indexLength > 0 ? new RestrictedStream(this.source, indexLength) : null) { yield return new PackAndIndex(packData, indexData, timestamp); } diff --git a/GVFS/GVFS.Common/BatchedLooseObjects/RestrictedStream.cs b/GVFS/GVFS.Common/NetworkStreams/RestrictedStream.cs similarity index 90% rename from GVFS/GVFS.Common/BatchedLooseObjects/RestrictedStream.cs rename to GVFS/GVFS.Common/NetworkStreams/RestrictedStream.cs index 212eb14494..6a8069318c 100644 --- a/GVFS/GVFS.Common/BatchedLooseObjects/RestrictedStream.cs +++ b/GVFS/GVFS.Common/NetworkStreams/RestrictedStream.cs @@ -1,7 +1,7 @@ using System; using System.IO; -namespace GVFS.Common +namespace GVFS.Common.NetworkStreams { /// /// Stream wrapper for a length-limited subview of another stream. @@ -15,21 +15,11 @@ internal class RestrictedStream : Stream private long position; private bool closed; - public RestrictedStream(Stream stream, long offset, long length, bool leaveOpen = false) + public RestrictedStream(Stream stream, long length, bool leaveOpen = true) { this.stream = stream; this.length = length; this.leaveOpen = leaveOpen; - - if (offset != 0) - { - if (!this.stream.CanSeek) - { - throw new InvalidOperationException(); - } - - this.stream.Seek(offset, SeekOrigin.Current); - } } public override bool CanRead diff --git a/GVFS/GVFS.Common/Physical/FileSystem/StreamReaderExtensions.cs b/GVFS/GVFS.Common/NetworkStreams/StreamReaderExtensions.cs similarity index 94% rename from GVFS/GVFS.Common/Physical/FileSystem/StreamReaderExtensions.cs rename to GVFS/GVFS.Common/NetworkStreams/StreamReaderExtensions.cs index 03fbbcc642..467bb048dd 100644 --- a/GVFS/GVFS.Common/Physical/FileSystem/StreamReaderExtensions.cs +++ b/GVFS/GVFS.Common/NetworkStreams/StreamReaderExtensions.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.IO; -namespace GVFS.Common.Physical.FileSystem +namespace GVFS.Common.NetworkStreams { public static class StreamReaderExtensions { diff --git a/GVFS/GVFS.Common/Paths.cs b/GVFS/GVFS.Common/Paths.cs new file mode 100644 index 0000000000..86dc8a280a --- /dev/null +++ b/GVFS/GVFS.Common/Paths.cs @@ -0,0 +1,77 @@ +using System; +using System.IO; +using System.Linq; + +namespace GVFS.Common +{ + public static class Paths + { + public static string GetGVFSEnlistmentRoot(string directory) + { + return GetRoot(directory, GVFSConstants.DotGVFS.Root); + } + + public static string GetGitEnlistmentRoot(string directory) + { + return GetRoot(directory, GVFSConstants.DotGit.Root); + } + + public static string GetNamedPipeName(string enlistmentRoot) + { + return "GVFS_" + enlistmentRoot.ToUpper().Replace(':', '_'); + } + + public static string GetServiceDataRoot(string serviceName) + { + return Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData, Environment.SpecialFolderOption.Create), + "GVFS", + serviceName); + } + + public static string GetServiceLogsPath(string serviceName) + { + return Path.Combine(GetServiceDataRoot(serviceName), "Logs"); + } + + private static string GetRoot(string startingDirectory, string rootName) + { + startingDirectory = startingDirectory.TrimEnd(GVFSConstants.PathSeparator); + DirectoryInfo dirInfo; + + try + { + dirInfo = new DirectoryInfo(startingDirectory); + } + catch (Exception) + { + return null; + } + + while (dirInfo != null) + { + if (dirInfo.Exists) + { + DirectoryInfo[] dotGVFSDirs = new DirectoryInfo[0]; + + try + { + dotGVFSDirs = dirInfo.GetDirectories(rootName); + } + catch (IOException) + { + } + + if (dotGVFSDirs.Count() == 1) + { + return dirInfo.FullName; + } + } + + dirInfo = dirInfo.Parent; + } + + return null; + } + } +} diff --git a/GVFS/GVFS.Common/Physical/Git/EndianHelper.cs b/GVFS/GVFS.Common/Physical/Git/EndianHelper.cs deleted file mode 100644 index 64a79405f4..0000000000 --- a/GVFS/GVFS.Common/Physical/Git/EndianHelper.cs +++ /dev/null @@ -1,47 +0,0 @@ -namespace GVFS.Common.Physical.Git -{ - public static class EndianHelper - { - public static short Swap(short source) - { - return (short)Swap((ushort)source); - } - - public static int Swap(int source) - { - return (int)Swap((uint)source); - } - - public static long Swap(long source) - { - return (long)((ulong)source); - } - - public static ushort Swap(ushort source) - { - return (ushort)(((source & 0x000000FF) << 8) | - ((source & 0x0000FF00) >> 8)); - } - - public static uint Swap(uint source) - { - return ((source & 0x000000FF) << 24) - | ((source & 0x0000FF00) << 8) - | ((source & 0x00FF0000) >> 8) - | ((source & 0xFF000000) >> 24); - } - - public static ulong Swap(ulong source) - { - return - ((source & 0x00000000000000FF) << 56) - | ((source & 0x000000000000FF00) << 40) - | ((source & 0x0000000000FF0000) << 24) - | ((source & 0x00000000FF000000) << 8) - | ((source & 0x000000FF00000000) >> 8) - | ((source & 0x0000FF0000000000) >> 24) - | ((source & 0x00FF000000000000) >> 40) - | ((source & 0xFF00000000000000) >> 56); - } - } -} diff --git a/GVFS/GVFS.Common/Physical/RegistryUtils.cs b/GVFS/GVFS.Common/Physical/RegistryUtils.cs deleted file mode 100644 index 9830b7ea99..0000000000 --- a/GVFS/GVFS.Common/Physical/RegistryUtils.cs +++ /dev/null @@ -1,33 +0,0 @@ -using Microsoft.Win32; - -namespace GVFS.Common.Physical -{ - public class RegistryUtils - { - public static string GetStringFromRegistry(RegistryHive registryHive, string key, string valueName) - { - string value = GetStringFromRegistry(registryHive, key, valueName, RegistryView.Registry64); - if (value == null) - { - value = GetStringFromRegistry(registryHive, key, valueName, RegistryView.Registry32); - } - - return value; - } - - private static string GetStringFromRegistry(RegistryHive registryHive, string key, string valueName, RegistryView view) - { - RegistryKey localKey = RegistryKey.OpenBaseKey(registryHive, view); - var localKeySub = localKey.OpenSubKey(key); - - object value = localKeySub == null ? null : localKeySub.GetValue(valueName); - - if (value == null) - { - return null; - } - - return (string)value; - } - } -} diff --git a/GVFS/GVFS.Common/ProcessHelper.cs b/GVFS/GVFS.Common/ProcessHelper.cs index c21bb7c4f2..dac4147938 100644 --- a/GVFS/GVFS.Common/ProcessHelper.cs +++ b/GVFS/GVFS.Common/ProcessHelper.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Win32; +using System; using System.Diagnostics; using System.IO; using System.Linq; @@ -152,6 +153,32 @@ public static ProcessResult Run(ProcessStartInfo processInfo, string errorMsgDel } } + public static object GetValueFromRegistry(RegistryHive registryHive, string key, string valueName) + { + object value = GetValueFromRegistry(registryHive, key, valueName, RegistryView.Registry64); + if (value == null) + { + value = GetValueFromRegistry(registryHive, key, valueName, RegistryView.Registry32); + } + + return value; + } + + public static string GetStringFromRegistry(RegistryHive registryHive, string key, string valueName) + { + object value = GetValueFromRegistry(registryHive, key, valueName); + return value as string; + } + + private static object GetValueFromRegistry(RegistryHive registryHive, string key, string valueName, RegistryView view) + { + RegistryKey localKey = RegistryKey.OpenBaseKey(registryHive, view); + var localKeySub = localKey.OpenSubKey(key); + + object value = localKeySub == null ? null : localKeySub.GetValue(valueName); + return value; + } + private static string StartProcess(Process executingProcess) { executingProcess.Start(); diff --git a/GVFS/GVFS.Common/ProcessWatcher.cs b/GVFS/GVFS.Common/ProcessWatcher.cs index 117354c7f3..907b504568 100644 --- a/GVFS/GVFS.Common/ProcessWatcher.cs +++ b/GVFS/GVFS.Common/ProcessWatcher.cs @@ -1,8 +1,6 @@ using System; -using System.Collections.Generic; using System.Diagnostics; using System.Management; -using System.Threading; using System.Threading.Tasks; namespace GVFS.Common @@ -12,6 +10,9 @@ namespace GVFS.Common /// internal class ProcessWatcher : IDisposable { + private const string CommandParentExePrefix = "git"; + private const string GVFSExe = "gvfs.exe"; + private const string QueryTemplate = @"SELECT * FROM __InstanceDeletionEvent WITHIN 1 WHERE TargetInstance ISA 'Win32_Process' and TargetInstance.ProcessId = '{0}'"; private readonly object terminationLock = new object(); @@ -28,7 +29,7 @@ public ProcessWatcher(Action onProcessTerminated) this.watcher.EventArrived += this.EventArrived; } - public void WatchForTermination(int pid, string expectedExeNamePrefix) + public void WatchForTermination(int pid) { lock (this.terminationLock) { @@ -45,9 +46,7 @@ public void WatchForTermination(int pid, string expectedExeNamePrefix) if (this.pendingPid != null) { Process process; - if (ProcessHelper.TryGetProcess(this.pendingPid.Value, out process) && - process.MainModule.ModuleName.StartsWith(expectedExeNamePrefix, StringComparison.OrdinalIgnoreCase) && - process.MainModule.ModuleName.EndsWith(GVFSConstants.ExecutableExtension, StringComparison.OrdinalIgnoreCase)) + if (ProcessHelper.TryGetProcess(this.pendingPid.Value, out process) && this.ShouldWatchProcess(process)) { this.watcher.Start(); this.currentPid = this.pendingPid; @@ -91,6 +90,12 @@ public void Dispose() { if (this.watcher != null) { + if (this.currentPid != null) + { + this.watcher.Stop(); + this.currentPid = null; + } + this.watcher.Dispose(); this.watcher = null; } @@ -117,5 +122,13 @@ private void EventArrived(object sender, EventArrivedEventArgs e) } } } + + private bool ShouldWatchProcess(Process process) + { + return + (process.MainModule.ModuleName.StartsWith(CommandParentExePrefix, StringComparison.OrdinalIgnoreCase) && + process.MainModule.ModuleName.EndsWith(GVFSConstants.ExecutableExtension, StringComparison.OrdinalIgnoreCase)) || + process.MainModule.ModuleName.Equals(GVFSExe, StringComparison.OrdinalIgnoreCase); + } } } diff --git a/GVFS/GVFS.Common/Physical/RepoMetadata.cs b/GVFS/GVFS.Common/RepoMetadata.cs similarity index 86% rename from GVFS/GVFS.Common/Physical/RepoMetadata.cs rename to GVFS/GVFS.Common/RepoMetadata.cs index b05574c29b..d38b21a722 100644 --- a/GVFS/GVFS.Common/Physical/RepoMetadata.cs +++ b/GVFS/GVFS.Common/RepoMetadata.cs @@ -1,17 +1,12 @@ -using GVFS.Common.Tracing; -using Microsoft.Isam.Esent; +using Microsoft.Isam.Esent; using Microsoft.Isam.Esent.Collections.Generic; -using Microsoft.Isam.Esent.Interop; using System; using System.IO; -namespace GVFS.Common.Physical +namespace GVFS.Common { public class RepoMetadata : IDisposable { - private const string ProjectionInvalidKey = "ProjectionInvalid"; - private const string PlaceholdersNeedUpdateKey = "PlaceholdersNeedUpdate"; - private PersistentDictionary repoMetadata; public RepoMetadata(string dotGVFSPath) @@ -59,22 +54,22 @@ public void SaveCurrentDiskLayoutVersion() public void SetProjectionInvalid(bool invalid) { - this.SetInvalid(Columns.ProjectionInvalidKey, invalid); + this.SetInvalid(Keys.ProjectionInvalid, invalid); } public bool GetProjectionInvalid() { - return this.HasEntry(Columns.ProjectionInvalidKey); + return this.HasEntry(Keys.ProjectionInvalid); } public void SetPlaceholdersNeedUpdate(bool needUpdate) { - this.SetInvalid(PlaceholdersNeedUpdateKey, needUpdate); + this.SetInvalid(Keys.PlaceholdersNeedUpdate, needUpdate); } public bool GetPlaceholdersNeedUpdate() { - return this.HasEntry(PlaceholdersNeedUpdateKey); + return this.HasEntry(Keys.PlaceholdersNeedUpdate); } public void Dispose() @@ -132,18 +127,19 @@ private bool CheckDiskLayoutVersion(bool allowUpgrade, out string error) return DiskLayoutVersion.CheckDiskLayoutVersion(this.repoMetadata, allowUpgrade, out error); } - private static class Columns + private static class Keys { - public const string ProjectionInvalidKey = "ProjectionInvalid"; - public const string PlaceholdersInvalidKey = "PlaceholdersInvalid"; - public const string DiskLayoutVersionKey = "DiskLayoutVersion"; + public const string ProjectionInvalid = "ProjectionInvalid"; + public const string PlaceholdersInvalid = "PlaceholdersInvalid"; + public const string DiskLayoutVersion = "DiskLayoutVersion"; + public const string PlaceholdersNeedUpdate = "PlaceholdersNeedUpdate"; } private static class DiskLayoutVersion { // The current disk layout version. This number should be bumped whenever a disk format change is made // that would impact and older GVFS's ability to mount the repo - public const int CurrentDiskLayoutVersion = 7; + public const int CurrentDiskLayoutVersion = 8; public const string MissingVersionError = "Enlistment disk layout version not found, check if a breaking change has been made to GVFS since cloning this enlistment."; @@ -160,7 +156,7 @@ private static class DiskLayoutVersion public static void SaveCurrentDiskLayoutVersion(PersistentDictionary repoMetadata) { - repoMetadata[Columns.DiskLayoutVersionKey] = CurrentDiskLayoutVersion.ToString(); + repoMetadata[Keys.DiskLayoutVersion] = CurrentDiskLayoutVersion.ToString(); repoMetadata.Flush(); } @@ -172,7 +168,7 @@ public static bool TryGetOnDiskLayoutVersion(PersistentDictionary + /// Computes the next backoff value in seconds. + /// + /// + /// Current failed attempt using 1-based counting. (i.e. currentFailedAttempt should be 1 if the first attempt just failed + /// + /// Maximum allowed backoff + /// Time to backoff in seconds + /// Computed backoff is randomly adjusted by +- 10% to help prevent clients from hitting servers at the same time + public static double CalculateBackoffSeconds(int currentFailedAttempt, double maxBackoffSeconds, double exponentialBackoffBase = DefaultExponentialBackoffBase) + { + if (currentFailedAttempt <= 1) + { + return 0; + } + + // Exponential backoff + double backOffSeconds = Math.Min(Math.Pow(exponentialBackoffBase, currentFailedAttempt), maxBackoffSeconds); + + // Timeout usually happens when the server is overloaded. If we give all machines the same timeout they will all make + // another request at approximately the same time causing the problem to happen again and again. To avoid that we + // introduce a random timeout. To avoid scaling it too high or too low, it is +- 10% of the average backoff + backOffSeconds *= .9 + (ThreadLocalRandom.NextDouble() * .2); + return backOffSeconds; + } + } +} diff --git a/GVFS/GVFS.Common/RetryConfig.cs b/GVFS/GVFS.Common/RetryConfig.cs new file mode 100644 index 0000000000..cb1aea00f9 --- /dev/null +++ b/GVFS/GVFS.Common/RetryConfig.cs @@ -0,0 +1,177 @@ +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using Microsoft.Diagnostics.Tracing; +using System; +using System.Linq; + +namespace GVFS.Common +{ + public class RetryConfig + { + public const int DefaultMaxRetries = 6; + public const int DefaultTimeoutSeconds = 30; + public const int FetchAndCloneTimeoutMinutes = 10; + + private const string EtwArea = nameof(RetryConfig); + + private const string MaxRetriesConfig = "max-retries"; + private const string TimeoutSecondsConfig = "timeout-seconds"; + private const int MinRetries = 0; + + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(DefaultTimeoutSeconds); + + public RetryConfig(int maxRetries = DefaultMaxRetries) + : this(maxRetries, DefaultTimeout) + { + } + + public RetryConfig(int maxRetries, TimeSpan timeout) + { + this.MaxRetries = maxRetries; + this.Timeout = timeout; + } + + public int MaxRetries { get; } + public int MaxAttempts + { + get { return this.MaxRetries + 1; } + } + + public TimeSpan Timeout { get; set; } + + public static bool TryLoadFromGitConfig(ITracer tracer, Enlistment enlistment, out RetryConfig retryConfig, out string error) + { + return TryLoadFromGitConfig(tracer, new GitProcess(enlistment), out retryConfig, out error); + } + + public static bool TryLoadFromGitConfig(ITracer tracer, GitProcess git, out RetryConfig retryConfig, out string error) + { + // TODO 1026787: Log failures to event log + + retryConfig = null; + + int maxRetries; + if (!TryLoadMaxRetries(git, out maxRetries, out error)) + { + if (tracer != null) + { + tracer.RelatedError( + new EventMetadata + { + { "Area", EtwArea }, + { "error", error }, + { "ErrorMessage", "TryLoadConfig: TryLoadMaxRetries failed" } + }); + } + + return false; + } + + TimeSpan timeout; + if (!TryLoadTimeout(git, out timeout, out error)) + { + if (tracer != null) + { + tracer.RelatedError( + new EventMetadata + { + { "Area", EtwArea }, + { "maxRetries", maxRetries }, + { "error", error }, + { "ErrorMessage", "TryLoadConfig: TryLoadTimeout failed" } + }); + } + + return false; + } + + retryConfig = new RetryConfig(maxRetries, timeout); + + if (tracer != null) + { + tracer.RelatedEvent( + EventLevel.Informational, + "RetryConfig_LoadedRetryConfig", + new EventMetadata + { + { "Area", EtwArea }, + { "Timeout", retryConfig.Timeout }, + { "MaxRetries", retryConfig.MaxRetries }, + { "Message", "RetryConfigLoaded" } + }); + } + + return true; + } + + private static bool TryLoadMaxRetries(GitProcess git, out int attempts, out string error) + { + return TryGetFromGitConfig( + git, + GVFSConstants.GitConfig.GVFSPrefix + MaxRetriesConfig, + DefaultMaxRetries, + MinRetries, + out attempts, + out error); + } + + private static bool TryLoadTimeout(GitProcess git, out TimeSpan timeout, out string error) + { + timeout = TimeSpan.FromSeconds(0); + int timeoutSeconds; + if (!TryGetFromGitConfig( + git, + GVFSConstants.GitConfig.GVFSPrefix + TimeoutSecondsConfig, + DefaultTimeoutSeconds, + 0, + out timeoutSeconds, + out error)) + { + return false; + } + + timeout = TimeSpan.FromSeconds(timeoutSeconds); + return true; + } + + private static bool TryGetFromGitConfig(GitProcess git, string configName, int defaultValue, int minValue, out int value, out string error) + { + value = defaultValue; + error = string.Empty; + + GitProcess.Result result = git.GetFromConfig(configName); + if (result.HasErrors) + { + if (result.Errors.Any()) + { + error = "Error while reading '" + configName + "' from config: " + result.Errors; + return false; + } + + // Git returns non-zero for non-existent settings and errors. + return true; + } + + string valueString = result.Output.TrimEnd('\n'); + if (string.IsNullOrWhiteSpace(valueString)) + { + // Use default value + return true; + } + + if (!int.TryParse(valueString, out value)) + { + error = string.Format("Misconfigured config setting {0}, could not parse value {1}", configName, valueString); + return false; + } + + if (value < minValue) + { + error = string.Format("Invalid value {0} for setting {1}, value must be greater than or equal to {2}", value, configName, minValue); + return false; + } + + return true; + } + } +} diff --git a/GVFS/GVFS.Common/RetryWrapper.cs b/GVFS/GVFS.Common/RetryWrapper.cs index aebf052793..55362af9da 100644 --- a/GVFS/GVFS.Common/RetryWrapper.cs +++ b/GVFS/GVFS.Common/RetryWrapper.cs @@ -4,24 +4,19 @@ using System.IO; using System.Net.Http; using System.Threading; -using System.Threading.Tasks; using System.Web; namespace GVFS.Common { public class RetryWrapper - { + { private const float MaxBackoffInSeconds = 300; // 5 minutes - private const float DefaultExponentialBackoffBase = 2; + private readonly int maxAttempts; + private readonly double exponentialBackoffBase; - private readonly int maxRetries; - private readonly float exponentialBackoffBase; - - private Random rng = new Random(); - - public RetryWrapper(int maxRetries, float exponentialBackoffBase = DefaultExponentialBackoffBase) + public RetryWrapper(int maxAttempts, double exponentialBackoffBase = RetryBackoff.DefaultExponentialBackoffBase) { - this.maxRetries = maxRetries; + this.maxAttempts = maxAttempts; this.exponentialBackoffBase = exponentialBackoffBase; } @@ -43,69 +38,12 @@ public static Action StandardErrorHandler(ITracer tracer, long r metadata["ErrorMessage"] = eArgs.Error != null ? eArgs.Error.ToString() : null; tracer.RelatedEvent(EventLevel.Verbose, JsonEtwTracer.NetworkErrorEventName, metadata, Keywords.Network); }; - } - - public async Task InvokeAsync(Func> toInvoke) - { - // Use 1-based counting. This makes reporting look a lot nicer and saves a lot of +1s - for (int tryCount = 1; tryCount <= this.maxRetries; ++tryCount) - { - try - { - CallbackResult result = await toInvoke(tryCount); - if (result.HasErrors) - { - if (!this.ShouldRetry(tryCount, null, result)) - { - return new InvocationResult(tryCount, result.Error, result.Result); - } - } - else - { - return new InvocationResult(tryCount, true, result.Result); - } - } - catch (Exception e) - { - Exception exceptionToReport = - e is AggregateException - ? ((AggregateException)e).Flatten().InnerException - : e; - - if (!this.IsHandlableException(exceptionToReport)) - { - throw; - } - - if (!this.ShouldRetry(tryCount, exceptionToReport, null)) - { - return new InvocationResult(tryCount, exceptionToReport); - } - } - - // Don't wait for the first retry, since it might just be transient. - // Don't wait after the last try. tryCount is 1-based, so last attempt is tryCount == maxRetries - if (tryCount > 1 && tryCount < this.maxRetries) - { - // Exponential backoff - double backOffSeconds = Math.Min(Math.Pow(this.exponentialBackoffBase, tryCount), MaxBackoffInSeconds); - - // Timeout usually happens when the server is overloaded. If we give all machines the same timeout they will all make - // another request at approximately the same time causing the problem to happen again and again. To avoid that we - // introduce a random timeout. To avoid scaling it too high or too low, it is +- 10% of the average backoff - backOffSeconds *= .9 + (this.rng.NextDouble() * .2); - await Task.Delay(TimeSpan.FromSeconds(backOffSeconds)); - } - } - - // This shouldn't be hit because ShouldRetry will cause a more useful message first. - return new InvocationResult(this.maxRetries, new Exception("Unexpected failure after retrying")); - } + } public InvocationResult Invoke(Func toInvoke) { // Use 1-based counting. This makes reporting look a lot nicer and saves a lot of +1s - for (int tryCount = 1; tryCount <= this.maxRetries; ++tryCount) + for (int tryCount = 1; tryCount <= this.maxAttempts; ++tryCount) { try { @@ -141,22 +79,16 @@ public InvocationResult Invoke(Func toInvoke) } // Don't wait for the first retry, since it might just be transient. - // Don't wait after the last try. tryCount is 1-based, so last attempt is tryCount == maxRetries - if (tryCount > 1 && tryCount < this.maxRetries) + // Don't wait after the last try. tryCount is 1-based, so last attempt is tryCount == maxAttempts + if (tryCount > 1 && tryCount < this.maxAttempts) { - // Exponential backoff - double backOffSeconds = Math.Min(Math.Pow(this.exponentialBackoffBase, tryCount), MaxBackoffInSeconds); - - // Timeout usually happens when the server is overloaded. If we give all machines the same timeout they will all make - // another request at approximately the same time causing the problem to happen again and again. To avoid that we - // introduce a random timeout. To avoid scaling it too high or too low, it is +- 10% of the average backoff - backOffSeconds *= .9 + (this.rng.NextDouble() * .2); + double backOffSeconds = RetryBackoff.CalculateBackoffSeconds(tryCount, MaxBackoffInSeconds, this.exponentialBackoffBase); Thread.Sleep(TimeSpan.FromSeconds(backOffSeconds)); } } // This shouldn't be hit because ShouldRetry will cause a more useful message first. - return new InvocationResult(this.maxRetries, new Exception("Unexpected failure after retrying")); + return new InvocationResult(this.maxAttempts, new Exception("Unexpected failure after retrying")); } private bool IsHandlableException(Exception e) @@ -170,7 +102,7 @@ private bool IsHandlableException(Exception e) private bool ShouldRetry(int tryCount, Exception e, CallbackResult result) { - bool willRetry = tryCount < this.maxRetries && + bool willRetry = tryCount < this.maxAttempts && (result == null || result.ShouldRetry); if (e != null) diff --git a/GVFS/GVFS.Common/SHA1Util.cs b/GVFS/GVFS.Common/SHA1Util.cs index bc6c3830c5..945b7a8c9e 100644 --- a/GVFS/GVFS.Common/SHA1Util.cs +++ b/GVFS/GVFS.Common/SHA1Util.cs @@ -1,5 +1,4 @@ -using System; -using System.Linq; +using System.Linq; using System.Security.Cryptography; using System.Text; @@ -7,6 +6,11 @@ namespace GVFS.Common { public static class SHA1Util { + public static bool IsValidShaFormat(string sha) + { + return sha.Length == 40 && !sha.Any(c => !(c >= '0' && c <= '9') && !(c >= 'a' && c <= 'f') && !(c >= 'A' && c <= 'F')); + } + public static string SHA1HashStringForUTF8String(string s) { return HexStringFromBytes(SHA1ForUTF8String(s)); diff --git a/GVFS/GVFS.Common/StreamUtil.cs b/GVFS/GVFS.Common/StreamUtil.cs index 04c93d8a8a..36602bc5f4 100644 --- a/GVFS/GVFS.Common/StreamUtil.cs +++ b/GVFS/GVFS.Common/StreamUtil.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; namespace GVFS.Common diff --git a/GVFS/GVFS.Common/Tracing/EventMetadata.cs b/GVFS/GVFS.Common/Tracing/EventMetadata.cs index ac9016c1d8..73f51de0bd 100644 --- a/GVFS/GVFS.Common/Tracing/EventMetadata.cs +++ b/GVFS/GVFS.Common/Tracing/EventMetadata.cs @@ -6,5 +6,13 @@ namespace GVFS.Common.Tracing // It's more obvious to see EventMetadata than Dictionary everywhere. public class EventMetadata : Dictionary { + public EventMetadata() + { + } + + public EventMetadata(Dictionary metadata) + : base(metadata) + { + } } } diff --git a/GVFS/GVFS.Common/Tracing/InProcEventListener.cs b/GVFS/GVFS.Common/Tracing/InProcEventListener.cs index e14611d6ed..ac2a969ab1 100644 --- a/GVFS/GVFS.Common/Tracing/InProcEventListener.cs +++ b/GVFS/GVFS.Common/Tracing/InProcEventListener.cs @@ -1,7 +1,5 @@ using Microsoft.Diagnostics.Tracing; using System; -using System.Collections.Generic; -using System.Linq; using System.Text; namespace GVFS.Common.Tracing diff --git a/GVFS/GVFS.Common/WindowsProcessJob.cs b/GVFS/GVFS.Common/WindowsProcessJob.cs deleted file mode 100644 index b42d0a8e78..0000000000 --- a/GVFS/GVFS.Common/WindowsProcessJob.cs +++ /dev/null @@ -1,145 +0,0 @@ -using Microsoft.Win32.SafeHandles; -using System; -using System.Diagnostics; -using System.Runtime.InteropServices; - -namespace GVFS.Common -{ - public class WindowsProcessJob : IDisposable - { - private SafeJobHandle jobHandle; - private bool disposed; - - public WindowsProcessJob(Process process) - { - IntPtr newHandle = Native.CreateJobObject(IntPtr.Zero, null); - if (newHandle == IntPtr.Zero) - { - throw new InvalidOperationException("Unable to create a job. Error: " + Marshal.GetLastWin32Error()); - } - - this.jobHandle = new SafeJobHandle(newHandle); - - Native.JOBOBJECT_BASIC_LIMIT_INFORMATION info = new Native.JOBOBJECT_BASIC_LIMIT_INFORMATION - { - LimitFlags = 0x2000 // JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - }; - - Native.JOBOBJECT_EXTENDED_LIMIT_INFORMATION extendedInfo = new Native.JOBOBJECT_EXTENDED_LIMIT_INFORMATION - { - BasicLimitInformation = info - }; - - int length = Marshal.SizeOf(typeof(Native.JOBOBJECT_EXTENDED_LIMIT_INFORMATION)); - if (!Native.SetInformationJobObject(this.jobHandle, Native.JobObjectInfoType.ExtendedLimitInformation, ref extendedInfo, (uint)length)) - { - throw new InvalidOperationException("Unable to configure the job. Error: " + Marshal.GetLastWin32Error()); - } - - if (!Native.AssignProcessToJobObject(this.jobHandle, process.Handle)) - { - throw new InvalidOperationException("Unable to add process to the job. Error: " + Marshal.GetLastWin32Error()); - } - } - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) - { - if (!this.disposed) - { - this.jobHandle.Dispose(); - this.jobHandle = null; - - this.disposed = true; - } - } - - private static class Native - { - public enum JobObjectInfoType - { - AssociateCompletionPortInformation = 7, - BasicLimitInformation = 2, - BasicUIRestrictions = 4, - EndOfJobTimeInformation = 6, - ExtendedLimitInformation = 9, - SecurityLimitInformation = 5, - GroupInformation = 11 - } - - [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - public static extern IntPtr CreateJobObject(IntPtr attributes, string name); - - [DllImport("kernel32.dll", SetLastError = true)] - public static extern bool SetInformationJobObject(SafeJobHandle jobHandle, JobObjectInfoType infoType, [In] ref JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobObjectInfo, uint jobObjectInfoLength); - - [DllImport("kernel32.dll", SetLastError = true)] - public static extern bool AssignProcessToJobObject(SafeJobHandle jobHandle, IntPtr processHandle); - - [DllImport("kernel32.dll", SetLastError = true)] - public static extern bool CloseHandle(IntPtr handle); - - [StructLayout(LayoutKind.Sequential)] - public struct IO_COUNTERS - { - public ulong ReadOperationCount; - public ulong WriteOperationCount; - public ulong OtherOperationCount; - public ulong ReadTransferCount; - public ulong WriteTransferCount; - public ulong OtherTransferCount; - } - - [StructLayout(LayoutKind.Sequential)] - public struct JOBOBJECT_BASIC_LIMIT_INFORMATION - { - public long PerProcessUserTimeLimit; - public long PerJobUserTimeLimit; - public uint LimitFlags; - public UIntPtr MinimumWorkingSetSize; - public UIntPtr MaximumWorkingSetSize; - public uint ActiveProcessLimit; - public UIntPtr Affinity; - public uint PriorityClass; - public uint SchedulingClass; - } - - [StructLayout(LayoutKind.Sequential)] - public struct SECURITY_ATTRIBUTES - { - public uint Length; - public IntPtr SecurityDescriptor; - public int InheritHandle; - } - - [StructLayout(LayoutKind.Sequential)] - public struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION - { - public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation; - public IO_COUNTERS IoInfo; - public UIntPtr ProcessMemoryLimit; - public UIntPtr JobMemoryLimit; - public UIntPtr PeakProcessMemoryUsed; - public UIntPtr PeakJobMemoryUsed; - } - } - - private sealed class SafeJobHandle : SafeHandleZeroOrMinusOneIsInvalid - { - public SafeJobHandle(IntPtr handle) : base(true) - { - this.SetHandle(handle); - } - - protected override bool ReleaseHandle() - { - return Native.CloseHandle(this.handle); - } - } - } -} \ No newline at end of file diff --git a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj index bb7fe14a1e..82f3e40c6a 100644 --- a/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj +++ b/GVFS/GVFS.FunctionalTests/GVFS.FunctionalTests.csproj @@ -91,9 +91,11 @@ True Settings.settings + + diff --git a/GVFS/GVFS.FunctionalTests/Properties/Settings.Designer.cs b/GVFS/GVFS.FunctionalTests/Properties/Settings.Designer.cs index 0439ced260..965fb7411e 100644 --- a/GVFS/GVFS.FunctionalTests/Properties/Settings.Designer.cs +++ b/GVFS/GVFS.FunctionalTests/Properties/Settings.Designer.cs @@ -88,7 +88,7 @@ internal sealed partial class Settings : global::System.Configuration.Applicatio [global::System.Configuration.ApplicationScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("FunctionalTests/20170602")] + [global::System.Configuration.DefaultSettingValueAttribute("FunctionalTests/20170707")] public string Commitish { get { return ((string)(this["Commitish"])); diff --git a/GVFS/GVFS.FunctionalTests/Properties/Settings.settings b/GVFS/GVFS.FunctionalTests/Properties/Settings.settings index c87f78fd5f..a49f15c36b 100644 --- a/GVFS/GVFS.FunctionalTests/Properties/Settings.settings +++ b/GVFS/GVFS.FunctionalTests/Properties/Settings.settings @@ -24,7 +24,7 @@ C:\Program Files\Git\bin\bash.exe - FunctionalTests/20170602 + FunctionalTests/20170707 C:\Repos\GVFSFunctionalTests\ControlRepo diff --git a/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs b/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs index 872c587abe..eb09cd32d3 100644 --- a/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs +++ b/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs @@ -53,8 +53,8 @@ public static DirectoryAdapter ShouldBeADirectory(this FileSystemInfo fileSystem public static string ShouldNotExistOnDisk(this string path, FileSystemRunner runner) { - runner.FileExists(path).ShouldEqual(false); - runner.DirectoryExists(path).ShouldEqual(false); + runner.FileExists(path).ShouldEqual(false, "File " + path + " exists when it should not"); + runner.DirectoryExists(path).ShouldEqual(false, "Directory " + path + " exists when it should not"); return path; } @@ -68,7 +68,7 @@ public class FileAdapter public FileAdapter(string path, FileSystemRunner runner) { this.runner = runner; - this.runner.FileExists(path).ShouldEqual(true, string.Format("Path does NOT exist: {0}", path)); + this.runner.FileExists(path).ShouldEqual(true, "Path does NOT exist: " + path); this.Path = path; } @@ -84,7 +84,7 @@ public string WithContents() public FileAdapter WithContents(string expectedContents) { - this.runner.ReadAllText(this.Path).ShouldEqual(expectedContents); + this.runner.ReadAllText(this.Path).ShouldEqual(expectedContents, "The contents of " + this.Path + " do not match what was expected"); return this; } @@ -93,16 +93,17 @@ public FileAdapter WithCaseMatchingName(string expectedName) FileInfo fileInfo = new FileInfo(this.Path); string parentPath = System.IO.Path.GetDirectoryName(this.Path); DirectoryInfo parentInfo = new DirectoryInfo(parentPath); - Assert.AreEqual(expectedName.Equals(parentInfo.GetFileSystemInfos(fileInfo.Name)[0].Name, StringComparison.Ordinal), true); + expectedName.Equals(parentInfo.GetFileSystemInfos(fileInfo.Name)[0].Name, StringComparison.Ordinal) + .ShouldEqual(true, this.Path + " does not have the correct case"); return this; } public FileInfo WithInfo(DateTime creation, DateTime lastWrite, DateTime lastAccess) { FileInfo info = new FileInfo(this.Path); - info.CreationTime.ShouldEqual(creation); - info.LastAccessTime.ShouldEqual(lastAccess); - info.LastWriteTime.ShouldEqual(lastWrite); + info.CreationTime.ShouldEqual(creation, "Creation time does not match"); + info.LastAccessTime.ShouldEqual(lastAccess, "Last access time does not match"); + info.LastWriteTime.ShouldEqual(lastWrite, "Last write time does not match"); return info; } @@ -110,21 +111,21 @@ public FileInfo WithInfo(DateTime creation, DateTime lastWrite, DateTime lastAcc public FileInfo WithInfo(DateTime creation, DateTime lastWrite, DateTime lastAccess, FileAttributes attributes) { FileInfo info = this.WithInfo(creation, lastWrite, lastAccess); - info.Attributes.ShouldEqual(attributes); + info.Attributes.ShouldEqual(attributes, "Attributes do not match"); return info; } public FileInfo WithAttribute(FileAttributes attribute) { FileInfo info = new FileInfo(this.Path); - info.Attributes.HasFlag(attribute).ShouldEqual(true); + info.Attributes.HasFlag(attribute).ShouldEqual(true, "Attributes do not have correct flag: " + attribute); return info; } public FileInfo WithoutAttribute(FileAttributes attribute) { FileInfo info = new FileInfo(this.Path); - info.Attributes.HasFlag(attribute).ShouldEqual(false); + info.Attributes.HasFlag(attribute).ShouldEqual(false, "Attributes have incorrect flag: " + attribute); return info; } } @@ -136,7 +137,7 @@ public class DirectoryAdapter public DirectoryAdapter(string path, FileSystemRunner runner) { this.runner = runner; - this.runner.DirectoryExists(path).ShouldEqual(true); + this.runner.DirectoryExists(path).ShouldEqual(true, "Directory " + path + " does not exist"); this.Path = path; } @@ -147,12 +148,12 @@ public string Path public void WithNoItems() { - Directory.EnumerateFileSystemEntries(this.Path).ShouldBeEmpty(); + Directory.EnumerateFileSystemEntries(this.Path).ShouldBeEmpty(this.Path + " is not empty"); } public void WithNoItems(string searchPattern) { - Directory.EnumerateFileSystemEntries(this.Path, searchPattern).ShouldBeEmpty(); + Directory.EnumerateFileSystemEntries(this.Path, searchPattern).ShouldBeEmpty(this.Path + " is not empty"); } public FileSystemInfo WithOneItem() @@ -163,7 +164,7 @@ public FileSystemInfo WithOneItem() public IEnumerable WithItems(int expectedCount) { IEnumerable items = this.WithItems(); - items.Count().ShouldEqual(expectedCount); + items.Count().ShouldEqual(expectedCount, this.Path + " has an invalid number of items"); return items; } @@ -176,7 +177,7 @@ public IEnumerable WithItems(string searchPattern) { DirectoryInfo directory = new DirectoryInfo(this.Path); IEnumerable items = directory.GetFileSystemInfos(searchPattern); - items.Any().ShouldEqual(true); + items.Any().ShouldEqual(true, this.Path + " does not have any items"); return items; } @@ -197,16 +198,17 @@ public DirectoryAdapter WithCaseMatchingName(string expectedName) DirectoryInfo info = new DirectoryInfo(this.Path); string parentPath = System.IO.Path.GetDirectoryName(this.Path); DirectoryInfo parentInfo = new DirectoryInfo(parentPath); - Assert.AreEqual(expectedName.Equals(parentInfo.GetDirectories(info.Name)[0].Name, StringComparison.Ordinal), true); + expectedName.Equals(parentInfo.GetDirectories(info.Name)[0].Name, StringComparison.Ordinal) + .ShouldEqual(true, this.Path + " does not have the correct case"); return this; } public DirectoryInfo WithInfo(DateTime creation, DateTime lastWrite, DateTime lastAccess) { DirectoryInfo info = new DirectoryInfo(this.Path); - info.CreationTime.ShouldEqual(creation); - info.LastAccessTime.ShouldEqual(lastAccess); - info.LastWriteTime.ShouldEqual(lastWrite); + info.CreationTime.ShouldEqual(creation, "Creation time does not match"); + info.LastAccessTime.ShouldEqual(lastAccess, "Last access time does not match"); + info.LastWriteTime.ShouldEqual(lastWrite, "Last write time does not match"); return info; } @@ -217,11 +219,11 @@ public DirectoryInfo WithInfo(DateTime creation, DateTime lastWrite, DateTime la if (ignoreRecallAttributes) { FileAttributes attributesWithoutRecall = info.Attributes & (FileAttributes)~(FileAttributeRecallOnOpen | FileAttributeRecallOnDataAccess); - attributesWithoutRecall.ShouldEqual(attributes); + attributesWithoutRecall.ShouldEqual(attributes, "Attributes do not match"); } else { - info.Attributes.ShouldEqual(attributes); + info.Attributes.ShouldEqual(attributes, "Attributes do not match"); } return info; @@ -230,7 +232,7 @@ public DirectoryInfo WithInfo(DateTime creation, DateTime lastWrite, DateTime la public DirectoryInfo WithAttribute(FileAttributes attribute) { DirectoryInfo info = new DirectoryInfo(this.Path); - info.Attributes.HasFlag(attribute).ShouldEqual(true); + info.Attributes.HasFlag(attribute).ShouldEqual(true, "Attributes do not have correct flag: " + attribute); return info; } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs new file mode 100644 index 0000000000..7a9ce0ff49 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs @@ -0,0 +1,35 @@ +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + [TestFixture] + public class CacheServerTests : TestsWithEnlistmentPerFixture + { + [TestCase] + public void SettingGitConfigChangesCacheServer() + { + const string ExpectedUrl = "https://myCache"; + + ProcessResult result = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "config gvfs.cache-server " + ExpectedUrl); + result.ExitCode.ShouldEqual(0, result.Errors); + + string getoutput = this.Enlistment.CacheServer("--get"); + getoutput.ShouldNotBeNull(); + string currentCache = getoutput.Trim().Substring(getoutput.LastIndexOf('\t') + 1); + currentCache.ShouldContain(ExpectedUrl); + } + + [TestCase] + public void SettingACacheReflectsChangesInCacheServerGet() + { + string getOutput = this.Enlistment.CacheServer("--get"); + getOutput.ShouldNotBeNull(); + + this.Enlistment.CacheServer("--set https://fake"); + + this.Enlistment.CacheServer("--get").ShouldNotEqual(getOutput); + } + } +} diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index e0d14c7377..4acca48b1c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -153,16 +153,20 @@ public void GitStatusAfterFileDelete() [TestCase, Order(8)] public void GitWithEnvironmentVariables() { - string previous = Environment.GetEnvironmentVariable("GIT_TRACE_PERFORMANCE"); + string previousPerf = Environment.GetEnvironmentVariable("GIT_TRACE_PERFORMANCE"); Environment.SetEnvironmentVariable("GIT_TRACE_PERFORMANCE", "1"); + string previousTrace = Environment.GetEnvironmentVariable("git_trace"); + Environment.SetEnvironmentVariable("git_trace", "1"); + // The trace info is an error, so we can't use CheckGitCommand(). // We just want to make sure this doesn't throw an exception. ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "branch", cleanErrors: false); result.Output.ShouldContain("* FunctionalTests"); result.Errors.ShouldNotContain(ignoreCase: true, unexpectedSubstrings: "exception"); result.Errors.ShouldContain("trace.c:", "git command:"); - Environment.SetEnvironmentVariable("GIT_TRACE_PERFORMANCE", previous); + Environment.SetEnvironmentVariable("GIT_TRACE_PERFORMANCE", previousPerf); + Environment.SetEnvironmentVariable("git_trace", previousTrace); } [TestCase, Order(9)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs new file mode 100644 index 0000000000..b503337c09 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs @@ -0,0 +1,91 @@ +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; +using System.Diagnostics; +using System.IO; +using System.Threading; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + [TestFixture] + public class UnmountTests : TestsWithEnlistmentPerFixture + { + private const int AlwaysExcludeOnDiskVersion = 5; + private const int GVFSGenericError = 3; + private const uint GenericRead = 2147483648; + private const uint FileFlagBackupSemantics = 3355443; + private const string IndexLockPath = ".git\\index.lock"; + private const string ExcludePath = ".git\\info\\exclude"; + private const string AlwaysExcludePath = ".git\\info\\always_exclude"; + + private FileSystemRunner fileSystem; + + public UnmountTests() + { + this.fileSystem = new SystemIORunner(); + } + + [SetUp] + public void SetupTest() + { + GVFSProcess gvfsProcess = new GVFSProcess( + Path.Combine(TestContext.CurrentContext.TestDirectory, Properties.Settings.Default.PathToGVFS), + this.Enlistment.EnlistmentRoot); + + if (!gvfsProcess.IsEnlistmentMounted()) + { + gvfsProcess.Mount(); + } + } + + [TestCase] + public void UnmountWaitsForLock() + { + ManualResetEventSlim lockHolder = GitHelpers.AcquireGVFSLock(this.Enlistment); + + using (Process unmountingProcess = this.StartUnmount()) + { + unmountingProcess.WaitForExit(3000).ShouldEqual(false, "Unmount completed while lock was acquired."); + + // Release the lock. + lockHolder.Set(); + + unmountingProcess.WaitForExit(10000).ShouldEqual(true, "Unmount didn't complete as expected."); + } + } + + [TestCase] + public void UnmountSkipLock() + { + ManualResetEventSlim lockHolder = GitHelpers.AcquireGVFSLock(this.Enlistment); + + using (Process unmountingProcess = this.StartUnmount("--skip-wait-for-lock")) + { + unmountingProcess.WaitForExit(10000).ShouldEqual(true, "Unmount didn't complete as expected."); + } + + // Signal process holding lock to terminate and release lock. + lockHolder.Set(); + } + + private Process StartUnmount(string extraParams = "") + { + string pathToGVFS = Path.Combine(TestContext.CurrentContext.TestDirectory, Properties.Settings.Default.PathToGVFS); + string enlistmentRoot = this.Enlistment.EnlistmentRoot; + + // TODO: 865304 Use app.config instead of --internal* arguments + ProcessStartInfo processInfo = new ProcessStartInfo(pathToGVFS); + processInfo.Arguments = "unmount " + extraParams + " --internal_use_only_service_name " + GVFSServiceProcess.TestServiceName; + processInfo.WindowStyle = ProcessWindowStyle.Hidden; + processInfo.WorkingDirectory = enlistmentRoot; + processInfo.UseShellExecute = false; + + Process executingProcess = new Process(); + executingProcess.StartInfo = processInfo; + executingProcess.Start(); + + return executingProcess; + } + } +} diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 16fb8cf297..f000cd3bec 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -464,6 +464,74 @@ public void FilterNonUTF8FileName() folderVirtualPath.ShouldBeADirectory(this.fileSystem).WithNoItems("ريلٌأكتوب.TXT"); } + [TestCase, Order(16)] + public void AllNullObjectRedownloaded() + { + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + this.Enlistment.Commitish); + ProcessResult revParseResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "rev-parse :Test_EPF_WorkingDirectoryTests/AllNullObjectRedownloaded.txt"); + string sha = revParseResult.Output.Trim(); + sha.Length.ShouldEqual(40); + string objectPath = Path.Combine(this.Enlistment.DotGVFSRoot, "gitObjectCache", sha.Substring(0, 2), sha.Substring(2, 38)); + objectPath.ShouldNotExistOnDisk(this.fileSystem); + + // At this point there should be no corrupt objects + string corruptObjectFolderPath = Path.Combine(this.Enlistment.DotGVFSRoot, "CorruptObjects"); + corruptObjectFolderPath.ShouldNotExistOnDisk(this.fileSystem); + + // Read a copy of AllNullObjectRedownloaded.txt to force the object to be downloaded + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "rev-parse :Test_EPF_WorkingDirectoryTests/AllNullObjectRedownloaded_copy.txt").Output.Trim().ShouldEqual(sha); + string testFileContents = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests\\AllNullObjectRedownloaded_copy.txt").ShouldBeAFile(this.fileSystem).WithContents(); + objectPath.ShouldBeAFile(this.fileSystem); + + // Set the contents of objectPath to all NULL + FileInfo objectFileInfo = new FileInfo(objectPath); + File.WriteAllBytes(objectPath, Enumerable.Repeat(0, (int)objectFileInfo.Length).ToArray()); + + // Read the original path and verify its contents are correct + this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests\\AllNullObjectRedownloaded.txt").ShouldBeAFile(this.fileSystem).WithContents(testFileContents); + + // Confirm there's a new item in the corrupt objects folder + corruptObjectFolderPath.ShouldBeADirectory(this.fileSystem); + FileSystemInfo badObject = corruptObjectFolderPath.ShouldBeADirectory(this.fileSystem).WithOneItem(); + (badObject as FileInfo).ShouldNotBeNull().Length.ShouldEqual(objectFileInfo.Length); + } + + [TestCase, Order(17)] + public void TruncatedObjectRedownloaded() + { + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + this.Enlistment.Commitish); + ProcessResult revParseResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "rev-parse :Test_EPF_WorkingDirectoryTests/TruncatedObjectRedownloaded.txt"); + string sha = revParseResult.Output.Trim(); + sha.Length.ShouldEqual(40); + string objectPath = Path.Combine(this.Enlistment.DotGVFSRoot, "gitObjectCache", sha.Substring(0, 2), sha.Substring(2, 38)); + objectPath.ShouldNotExistOnDisk(this.fileSystem); + + string corruptObjectFolderPath = Path.Combine(this.Enlistment.DotGVFSRoot, "CorruptObjects"); + int initialCorruptObjectCount = 0; + if (this.fileSystem.DirectoryExists(corruptObjectFolderPath)) + { + initialCorruptObjectCount = new DirectoryInfo(corruptObjectFolderPath).EnumerateFileSystemInfos().Count(); + } + + // Read a copy of TruncatedObjectRedownloaded.txt to force the object to be downloaded + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "rev-parse :Test_EPF_WorkingDirectoryTests/TruncatedObjectRedownloaded_copy.txt").Output.Trim().ShouldEqual(sha); + string testFileContents = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests\\TruncatedObjectRedownloaded_copy.txt").ShouldBeAFile(this.fileSystem).WithContents(); + objectPath.ShouldBeAFile(this.fileSystem); + + // Truncate the contents of objectPath + FileInfo objectFileInfo = new FileInfo(objectPath); + using (FileStream objectStream = new FileStream(objectPath, FileMode.Open)) + { + objectStream.SetLength(objectFileInfo.Length - 8); + } + + // Read the original path and verify its contents are correct + this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests\\TruncatedObjectRedownloaded.txt").ShouldBeAFile(this.fileSystem).WithContents(testFileContents); + + // Confirm there's a new item in the corrupt objects folder + corruptObjectFolderPath.ShouldBeADirectory(this.fileSystem).WithItems().Count().ShouldEqual(initialCorruptObjectCount + 1); + } + private void FolderEnumerationShouldHaveSingleEntry(string folderVirtualPath, string expectedEntryName, string searchPatten) { IEnumerable folderEntries; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs index 6d8f8d4647..46dd1dd759 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs @@ -1,4 +1,5 @@ -using GVFS.Tests.Should; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; using NUnit.Framework; using System.IO; @@ -13,7 +14,7 @@ public void FixesCorruptHeadSha() this.Enlistment.UnmountGVFS(); string headFilePath = Path.Combine(this.Enlistment.RepoRoot, ".git", "HEAD"); - File.WriteAllText(headFilePath, "000"); + File.WriteAllText(headFilePath, "0000"); this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when HEAD is corrupt"); @@ -44,7 +45,7 @@ public void FixesCorruptBlobSizesDatabase() // Most other files in an ESENT folder can be corrupted without blocking GVFS mount. ESENT just recreates them. string blobSizesDbPath = Path.Combine(this.Enlistment.DotGVFSRoot, "BlobSizes", "PersistentDictionary.edb"); - File.WriteAllText(blobSizesDbPath, "000"); + File.WriteAllText(blobSizesDbPath, "0000"); this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when blob size db is corrupt"); @@ -52,5 +53,38 @@ public void FixesCorruptBlobSizesDatabase() this.Enlistment.MountGVFS(); } + + [TestCase] + public void FixesMissingGitIndex() + { + this.Enlistment.UnmountGVFS(); + + string gitIndexPath = Path.Combine(this.Enlistment.RepoRoot, ".git", "index"); + File.Delete(gitIndexPath); + + this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when git index is missing"); + + this.Enlistment.Repair(); + + this.Enlistment.MountGVFS(); + } + + [TestCase] + public void FixesCorruptGitConfig() + { + this.Enlistment.UnmountGVFS(); + + string gitIndexPath = Path.Combine(this.Enlistment.RepoRoot, ".git", "config"); + File.WriteAllText(gitIndexPath, "[cor"); + + this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when git config is missing"); + + this.Enlistment.Repair(); + + ProcessResult result = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "remote add origin " + this.Enlistment.RepoUrl); + result.ExitCode.ShouldEqual(0, result.Errors); + + this.Enlistment.MountGVFS(); + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index a8638fa9e4..5952b077d4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -116,15 +116,33 @@ public void CanFetchAndCheckoutBranchIntoEmptyGitRepo() public void CanUpdateIndex() { // Testing index versions 2, 3 and 4. Not bothering to test version 1; it's not in use anymore. - this.CanUpdateIndex(2); - this.CanUpdateIndex(3); - this.CanUpdateIndex(4); + this.CanUpdateIndex(2, indexSigningOff: true); + this.CanUpdateIndex(3, indexSigningOff: true); + this.CanUpdateIndex(4, indexSigningOff: true); + + this.CanUpdateIndex(2, indexSigningOff: false); + this.CanUpdateIndex(3, indexSigningOff: false); + this.CanUpdateIndex(4, indexSigningOff: false); + } + + [TestCase] + public void CanFetchAndCheckoutAfterDeletingIndex() + { + this.RunFastFetch("--checkout -b master"); + + File.Delete(Path.Combine(this.fastFetchRepoRoot, ".git", "index")); + this.RunFastFetch("--checkout -b " + Settings.Default.Commitish); + + this.CurrentBranchShouldEqual(Settings.Default.Commitish); + + this.fastFetchRepoRoot.ShouldBeADirectory(FileSystemRunner.DefaultRunner) + .WithDeepStructure(FileSystemRunner.DefaultRunner, this.fastFetchControlRoot, skipEmptyDirectories: false); } - public void CanUpdateIndex(int indexVersion) - { + public void CanUpdateIndex(int indexVersion, bool indexSigningOff) + { // Initialize the repo - GitProcess.Invoke(this.fastFetchRepoRoot, "config --local --add core.gvfs 1"); + GitProcess.Invoke(this.fastFetchRepoRoot, "config --local --add core.gvfs " + (indexSigningOff ? 1 : 0)); this.CanFetchAndCheckoutBranchIntoEmptyGitRepo(); string lsfilesAfterFirstFetch = GitProcess.Invoke(this.fastFetchRepoRoot, "ls-files --debug"); lsfilesAfterFirstFetch.ShouldBeNonEmpty(); @@ -154,25 +172,7 @@ public void CanUpdateIndex(int indexVersion) // Verify that the final results are the same as the intial fetch results lsfilesAfterUpdate2.ShouldEqual(lsfilesAfterFirstFetch, "Incremental update should not change index"); } - - [TestCase] - public void ShouldNotUpdateIndex() - { - // Initialize the repo - GitProcess.Invoke(this.fastFetchRepoRoot, "config --local --add core.gvfs 0"); - this.CanFetchAndCheckoutBranchIntoEmptyGitRepo(); - - // Reset the index and get baseline. - GitProcess.Invoke(this.fastFetchRepoRoot, "read-tree HEAD"); - string lsfilesNeedUpdated = GitProcess.Invoke(this.fastFetchRepoRoot, "ls-files --debug"); - - // Fastfetch. Shouldn't update index without core.gvfs disabling signing. - string fastfetchoutput = this.RunFastFetch("--checkout --Allow-index-metadata-update-from-working-tree"); - Trace.WriteLine(fastfetchoutput); // Written to log file for manual investigation - string lsfilesAfterUpdate = GitProcess.Invoke(this.fastFetchRepoRoot, "ls-files --debug"); - lsfilesAfterUpdate.ShouldEqual(lsfilesNeedUpdated, "Update should be skipped without core.gvfs"); - } - + [TestCase] public void IncrementalChangesLeaveGoodStatus() { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index b379b3417d..2a82265c2c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -207,7 +207,7 @@ public void DeleteEmptyFolderPlaceholderAndCheckoutBranchThatDoesNotHaveFolder() this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); this.ShouldNotExistOnDisk(testFile); - this.ValidateGitCommand("checkout -b tests/functional/DeleteEmptyFolderPlaceholderAndCheckoutBranchThatHasFolder" + this.ControlGitRepo.Commitish); + this.ValidateGitCommand("checkout -b tests/functional/DeleteEmptyFolderPlaceholderAndCheckoutBranchThatDoesNotHaveFolder" + this.ControlGitRepo.Commitish); // Test_ConflictTests\AddedFiles will only be on disk in the GVFS enlistment, delete it there string virtualFolder = Path.Combine(this.Enlistment.RepoRoot, testFolder); @@ -300,5 +300,24 @@ public void ModifyAndCheckoutFirstOfSeveralFilesWhoseNamesAppearBeforeDot() this.ValidateGitCommand("status"); this.FileShouldHaveContents("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(a).txt", originalContent); } + + [TestCase] + public void ResetMixedToCommitWithNewFileThenCheckoutNewBranchAndCheckoutCommitWithNewFile() + { + this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); + + // Commit 170b13ce1990c53944403a70e93c257061598ae0 was prior to the additional of these + // three files in commit f2546f8e9ce7d7b1e3a0835932f0d6a6145665b1: + // Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt + // Test_ConflictTests/AddedFiles/AddedByBothSameContent.txt + // Test_ConflictTests/AddedFiles/AddedBySource.txt + this.ValidateGitCommand("checkout 170b13ce1990c53944403a70e93c257061598ae0"); + this.ValidateGitCommand("reset --mixed f2546f8e9ce7d7b1e3a0835932f0d6a6145665b1"); + + // Use RunGitCommand rather than ValidateGitCommand as G4W optimizations for "checkout -b" mean that the + // command will not report modified and deleted files + this.RunGitCommand("checkout -b tests/functional/ResetMixedToCommitWithNewFileThenCheckoutNewBranchAndCheckoutCommitWithNewFile"); + this.ValidateGitCommand("checkout f2546f8e9ce7d7b1e3a0835932f0d6a6145665b1"); + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 9ce9fce182..f1a94668d9 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -270,8 +270,7 @@ public void DeleteFileWithNameAheadOfDotAndSwitchCommits() [TestCase] public void AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() { - // 663045 - Confirm that folder can be deleted after adding a file then changing - // branches + // 663045 - Confirm that folder can be deleted after adding a file then changing branches string newFileParentFolderPath = @"GVFS\GVFS\CommandLine"; string newFilePath = newFileParentFolderPath + @"\testfile.txt"; string newFileContents = "test contents"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index 0916325f60..12328bb281 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -1,4 +1,5 @@ using GVFS.FunctionalTests.Category; +using GVFS.FunctionalTests.Should; using NUnit.Framework; namespace GVFS.FunctionalTests.Tests.GitCommands @@ -81,6 +82,30 @@ public void ResetMixedOnlyAddedThenCheckoutWithConflicts() this.FilesShouldMatchCheckoutOfTargetBranch(); } + [TestCase] + public void ResetMixedAndCheckoutFile() + { + this.ControlGitRepo.Fetch("FunctionalTests/20170602"); + + // We start with a branch that deleted two files that were present in its parent commit + this.ValidateGitCommand("checkout FunctionalTests/20170602"); + + // Then reset --mixed to the parent commit, and validate that the deleted files did not come back into the projection + this.ValidateGitCommand("reset --mixed HEAD~1"); + this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) + .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, skipEmptyDirectories: false); + + // And checkout a file (without changing branches) and ensure that that doesn't update the projection either + this.ValidateGitCommand("checkout HEAD~2 .gitattributes"); + this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) + .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, skipEmptyDirectories: false); + + // And now if we checkout the original commit, the deleted files should stay deleted + this.ValidateGitCommand("checkout FunctionalTests/20170602"); + this.Enlistment.RepoRoot.ShouldBeADirectory(this.FileSystem) + .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, skipEmptyDirectories: false); + } + protected override void CreateEnlistment() { base.CreateEnlistment(); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index f72a803c6c..48b0087f69 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -180,6 +180,11 @@ public void UnmountGVFS() this.gvfsProcess.Unmount(); } + public string CacheServer(string args) + { + return this.gvfsProcess.CacheServer(args); + } + public void UnmountAndDeleteAll() { this.UnmountGVFS(); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index 0d439df8b9..4f7692e345 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -34,9 +34,9 @@ public bool TryMount(out string output) { string mountCommand = "mount " + this.enlistmentRoot + " --internal_use_only_service_name " + GVFSServiceProcess.TestServiceName; - this.IsGVFSMounted().ShouldEqual(false, "GVFS is already mounted"); + this.IsEnlistmentMounted().ShouldEqual(false, "GVFS is already mounted"); output = this.CallGVFS(mountCommand); - return this.IsGVFSMounted(); + return this.IsEnlistmentMounted(); } public string Prefetch(string folderPath) @@ -72,6 +72,11 @@ public string Status() return this.CallGVFS("status " + this.enlistmentRoot); } + public string CacheServer(string args) + { + return this.CallGVFS("cache-server " + args + " " + this.enlistmentRoot); + } + public void Unmount() { string unmountArgs = string.Join( @@ -79,10 +84,10 @@ public void Unmount() "unmount " + this.enlistmentRoot, "--internal_use_only_service_name " + GVFSServiceProcess.TestServiceName); string result = this.CallGVFS(unmountArgs); - this.IsGVFSMounted().ShouldEqual(false, "GVFS did not unmount: " + result); + this.IsEnlistmentMounted().ShouldEqual(false, "GVFS did not unmount: " + result); } - private bool IsGVFSMounted() + public bool IsEnlistmentMounted() { string statusResult = this.CallGVFS("status " + this.enlistmentRoot); return statusResult.Contains("Mount status: Ready"); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSServiceProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSServiceProcess.cs index 7d00b20539..1bd82ef1a4 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSServiceProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSServiceProcess.cs @@ -1,7 +1,9 @@ using GVFS.Tests.Should; using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.ServiceProcess; using System.Threading; @@ -46,14 +48,15 @@ public static void StopService() { try { - using (ServiceController controller = new ServiceController(TestServiceName)) + ServiceController testService = ServiceController.GetServices().SingleOrDefault(service => service.ServiceName == TestServiceName); + if (testService != null) { - if (controller.Status == ServiceControllerStatus.Running) + if (testService.Status == ServiceControllerStatus.Running) { - controller.Stop(); + testService.Stop(); } - controller.WaitForStatus(ServiceControllerStatus.Stopped); + testService.WaitForStatus(ServiceControllerStatus.Stopped); } } catch (InvalidOperationException) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 50b823ec70..82ffad5c43 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -11,8 +11,6 @@ namespace GVFS.FunctionalTests.Tools public static class GitHelpers { public const string AlwaysExcludeFilePath = @".git\info\always_exclude"; - private const int MaxRetries = 10; - private const int ThreadSleepMS = 1500; public static void CheckGitCommand(string virtualRepoRoot, string command, params string[] expectedLinesInResult) { diff --git a/GVFS/GVFS.FunctionalTests/app.config b/GVFS/GVFS.FunctionalTests/app.config index 881571fa9c..c78228cc22 100644 --- a/GVFS/GVFS.FunctionalTests/app.config +++ b/GVFS/GVFS.FunctionalTests/app.config @@ -35,7 +35,7 @@ C:\Program Files\Git\bin\bash.exe - FunctionalTests/20170602 + FunctionalTests/20170707 C:\Repos\GVFSFunctionalTests\ControlRepo diff --git a/GVFS/GVFS.GVFlt/DotGit/GitIndexProjection.cs b/GVFS/GVFS.GVFlt/DotGit/GitIndexProjection.cs index 385035268d..213fd98663 100644 --- a/GVFS/GVFS.GVFlt/DotGit/GitIndexProjection.cs +++ b/GVFS/GVFS.GVFlt/DotGit/GitIndexProjection.cs @@ -1,9 +1,8 @@ -using GVFS.Common; +using GvFlt; +using GVFS.Common; +using GVFS.Common.Git; using GVFS.Common.NamedPipes; -using GVFS.Common.Physical; -using GVFS.Common.Physical.Git; using GVFS.Common.Tracing; -using GVFSGvFltWrapper; using Microsoft.Diagnostics.Tracing; using Microsoft.Isam.Esent.Collections.Generic; using System; @@ -16,6 +15,8 @@ using System.Threading; using System.Threading.Tasks; +using ITracer = GVFS.Common.Tracing.ITracer; + namespace GVFS.GVFlt.DotGit { public class GitIndexProjection : IDisposable, IProfilerOnlyIndexProjection @@ -28,7 +29,7 @@ public class GitIndexProjection : IDisposable, IProfilerOnlyIndexProjection public const string ProjectionIndexBackupName = "GVFS_projection"; - private const GvUpdateType PlaceholderUpdateFlags = GvUpdateType.UpdateAllowDirtyMetadata | GvUpdateType.UpdateAllowReadOnly; + private const UpdateType PlaceholderUpdateFlags = UpdateType.AllowDirtyMetadata | UpdateType.AllowReadOnly; private const string EtwArea = "GitIndexProjection"; @@ -40,7 +41,8 @@ public class GitIndexProjection : IDisposable, IProfilerOnlyIndexProjection private GVFSContext context; private RepoMetadata repoMetadata; - private GvFltWrapper gvflt; + private VirtualizationInstance gvflt; + private SparseCheckout sparseCheckout; private FileOrFolderData rootFolderData; @@ -55,8 +57,14 @@ public class GitIndexProjection : IDisposable, IProfilerOnlyIndexProjection private ManualResetEventSlim projectionParseComplete; private ManualResetEventSlim externalLockReleaseHandlingComplete; - private bool offsetsInvalid; + private bool offsetsInvalid; private bool projectionInvalid; + + // sparseCheckoutInvalid: If true, a change to the index that did not trigger a new projection + // has been made and GVFS has not yet validated that all entries whose skip-worktree bit is + // cleared are in the sparse-checkout + private bool sparseCheckoutInvalid; + private ConcurrentHashSet updatePlaceholderFailures; private ConcurrentHashSet deletePlaceholderFailures; @@ -80,7 +88,9 @@ public class GitIndexProjection : IDisposable, IProfilerOnlyIndexProjection GVFSGitObjects gitObjects, PersistentDictionary blobSizes, RepoMetadata repoMetadata, - GvFltWrapper gvflt) + VirtualizationInstance gvflt, + PersistentDictionary placeholderList, + SparseCheckout sparseCheckout) { this.context = context; this.gitObjects = gitObjects; @@ -94,7 +104,16 @@ public class GitIndexProjection : IDisposable, IProfilerOnlyIndexProjection this.projectionIndexBackupPath = Path.Combine(this.context.Enlistment.DotGVFSRoot, ProjectionIndexBackupName); this.indexPath = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Index); this.externalLockReleaseHandlingComplete = new ManualResetEventSlim(false); - this.placeholderList = new PersistentDictionary(Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.PlaceholderList)); + this.placeholderList = placeholderList; + this.sparseCheckout = sparseCheckout; + } + + private enum IndexAction + { + RebuildProjection, + UpdateOffsets, + ValidateSparseCheckout, + UpdateOffsetsAndValidateSparseCheckout } public int PlaceholderCount @@ -109,25 +128,53 @@ public int PlaceholderCount /// Force the index file to be parsed and a new projection collection to be built. /// This method should only be used to measure index parsing performance. /// - void IProfilerOnlyIndexProjection.ForceParseIndexFileForNewProjection() + void IProfilerOnlyIndexProjection.ForceRebuildProjection() + { + this.CopyIndexFileAndBuildProjection(); + } + + /// + /// Force the index file to be parsed to update offsets and validate the sparse checkout. + /// This method should only be used to measure index parsing performance. + /// + void IProfilerOnlyIndexProjection.ForceUpdateOffsetsAndValidateSparseCheckout() { - this.CopyIndexFileAndParse(); + using (FileStream indexStream = new FileStream(this.indexPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, IndexFileStreamBufferSize)) + { + this.TryIndexAction(indexStream, IndexAction.UpdateOffsetsAndValidateSparseCheckout); + } } /// - /// Force the index file to be parsed to update offsets. + /// Force the index file to be parsed to validate the sparse-checkout. /// This method should only be used to measure index parsing performance. /// - void IProfilerOnlyIndexProjection.ForceParseIndexToUpdateOffsets() + void IProfilerOnlyIndexProjection.ForceValidateSparseCheckout() { using (FileStream indexStream = new FileStream(this.indexPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, IndexFileStreamBufferSize)) { - this.ParseIndex(indexStream, updateOffsetsOnly: true); + this.TryIndexAction(indexStream, IndexAction.ValidateSparseCheckout); + } + } + + public void BuildProjectionFromPath(string indexPath) + { + using (FileStream indexStream = new FileStream(indexPath, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, IndexFileStreamBufferSize)) + { + this.ParseIndexAndBuildProjection(indexStream); } } public void Initialize(ReliableBackgroundOperations backgroundQueue) { + if (!File.Exists(this.indexPath)) + { + string message = "GVFS requires the .git\\index to exist"; + EventMetadata metadata = CreateErrorMetadata(message); + this.context.Tracer.RelatedError(metadata); + throw new FileNotFoundException(message); + } + this.gitIndexLock = new FileBasedLock( this.context.FileSystem, this.context.Tracer, @@ -145,13 +192,15 @@ public void Initialize(ReliableBackgroundOperations + /// Update the index file offsets that GVFS has cached in memory. + /// + /// + /// As a performance optimization, UpdateOffsets will also validate the sparse-checkout file (if it needs validation). + /// + public CallbackResult UpdateOffsets() + { + try + { + if (this.offsetsInvalid) + { if (this.lastUpdateTime == uint.MaxValue) { this.lastUpdateTime = 0; @@ -420,13 +509,47 @@ public CallbackResult ClearSkipWorktreeBit(string filePath) ++this.lastUpdateTime; } - this.ParseIndex(this.indexFileStream, updateOffsetsOnly: true); - - if (this.offsetsInvalid) + // Performance optimization: If sparseCheckoutInvalid is true, save GVFS from reading the index a second time by + // updating offsets and validating the sparse-checkout in a single pass + CallbackResult result = this.TryIndexAction( + this.indexFileStream, + this.sparseCheckoutInvalid ? IndexAction.UpdateOffsetsAndValidateSparseCheckout : IndexAction.UpdateOffsets); + if (result == CallbackResult.Success) { - return CallbackResult.RetryableError; + this.sparseCheckoutInvalid = false; + this.offsetsInvalid = false; } + + return result; } + } + catch (IOException e) + { + EventMetadata metadata = CreateErrorMetadata("IOException in UpdateOffsets (RetryableError)", e); + this.context.Tracer.RelatedError(metadata); + + return CallbackResult.RetryableError; + } + catch (Exception e) + { + EventMetadata metadata = CreateErrorMetadata("Exception in UpdateOffsets (FatalError)", e); + this.context.Tracer.RelatedError(metadata); + + return CallbackResult.FatalError; + } + + return CallbackResult.Success; + } + + public CallbackResult ClearSkipWorktreeBit(string filePath) + { + try + { + CallbackResult updateOffsetsResult = this.UpdateOffsets(); + if (updateOffsetsResult != CallbackResult.Success) + { + return updateOffsetsResult; + } long offset; if (this.TryGetIndexPathOffset(filePath, out offset)) @@ -438,14 +561,14 @@ public CallbackResult ClearSkipWorktreeBit(string filePath) } catch (IOException e) { - EventMetadata metadata = CreateErrorMetadata("IOException in ClearSkipWorktreeAndRemoveFromPlaceholderList (RetryableError)", e); + EventMetadata metadata = CreateErrorMetadata("IOException in ClearSkipWorktreeBit (RetryableError)", e); this.context.Tracer.RelatedError(metadata); return CallbackResult.RetryableError; } catch (Exception e) { - EventMetadata metadata = CreateErrorMetadata("Exception in ClearSkipWorktreeAndRemoveFromPlaceholderList (FatalError)", e); + EventMetadata metadata = CreateErrorMetadata("Exception in ClearSkipWorktreeBit (FatalError)", e); this.context.Tracer.RelatedError(metadata); return CallbackResult.FatalError; @@ -527,6 +650,10 @@ private static int ReadReplaceLength(Stream stream) for (int i = 0; (headerByte & 0x80) != 0; i++) { headerByte = stream.ReadByte(); + if (headerByte < 0) + { + throw new EndOfStreamException("Unexpected end of stream while reading git index."); + } offset += 1; offset = (offset << 7) + (headerByte & 0x7f); @@ -984,7 +1111,7 @@ private void ParseIndexThreadMain() try { this.lastUpdateTime = 0; - this.CopyIndexFileAndParse(); + this.CopyIndexFileAndBuildProjection(); } catch (Win32Exception e) { @@ -1077,6 +1204,7 @@ private void UpdatePlaceholders() placeholderUpdateThreads[i] = new Thread( () => { + // We have a top-level try\catch for any unhandled exceptions thrown in the newly created thread try { for (int j = start; j < end; ++j) @@ -1122,8 +1250,8 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) string projectedSha; if (!this.TryGetSha(childName, parentKey, out projectedSha)) { - GvUpdateFailureCause failureReason = GvUpdateFailureCause.NoFailure; - StatusCode status = this.gvflt.GvDeleteFile(virtualPath, PlaceholderUpdateFlags, ref failureReason); + UpdateFailureCause failureReason = UpdateFailureCause.NoFailure; + NtStatus status = this.gvflt.DeleteFile(virtualPath, PlaceholderUpdateFlags, ref failureReason); this.ProcessGvUpdateDeletePlaceholderResult(virtualPath, string.Empty, status, failureReason, deleteOperation: true); } else @@ -1132,13 +1260,13 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) if (!onDiskSha.Equals(projectedSha)) { DateTime now = DateTime.UtcNow; - GvUpdateFailureCause failureReason = GvUpdateFailureCause.NoFailure; - StatusCode status; + UpdateFailureCause failureReason = UpdateFailureCause.NoFailure; + NtStatus status; try { FileOrFolderData data = this.GetProjectedFileOrFolderData(childName, parentKey, populateSize: true); - status = this.gvflt.GvUpdatePlaceholderIfNeeded( + status = this.gvflt.UpdatePlaceholderIfNeeded( virtualPath, creationTime: now, lastAccessTime: now, @@ -1146,14 +1274,14 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) changeTime: now, fileAttributes: (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_ARCHIVE, endOfFile: data.Size, - contentId: projectedSha, - epochId: null, + contentId: GVFltCallbacks.ConvertShaToContentId(projectedSha), + epochId: GVFltCallbacks.GetEpochId(), updateFlags: PlaceholderUpdateFlags, failureReason: ref failureReason); } catch (Exception e) { - status = StatusCode.StatusUnsuccessful; + status = NtStatus.Unsuccessful; EventMetadata metadata = CreateEventMetadata("UpdateOrDeletePlaceholder: Exception while trying to update placeholder", e); metadata.Add("virtualPath", virtualPath); @@ -1168,14 +1296,14 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) private void ProcessGvUpdateDeletePlaceholderResult( string virtualPath, string projectedSha, - StatusCode status, - GvUpdateFailureCause failureReason, + NtStatus status, + UpdateFailureCause failureReason, bool deleteOperation) { EventMetadata metadata; switch (status) { - case StatusCode.StatusSucccess: + case NtStatus.Succcess: if (deleteOperation) { this.placeholderList.Remove(virtualPath); @@ -1187,7 +1315,7 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) break; - case StatusCode.StatusIoReparseTagNotHandled: + case NtStatus.IoReparseTagNotHandled: // Attempted to update\delete a file that has a non-GvFlt reparse point metadata = CreateEventMetadata("UpdateOrDeletePlaceholder: StatusIoReparseTagNotHandled"); metadata.Add("deleteOperation", deleteOperation); @@ -1198,7 +1326,7 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) this.placeholderList.Remove(virtualPath); break; - case StatusCode.StatusFileSystemVirtualizationInvalidOperation: + case NtStatus.FileSystemVirtualizationInvalidOperation: // GVFS attempted to update\delete a file that is no longer partial. // This can occur if a file is converted from partial to full (or tombstone) while a git command is running // Any tasks scheduled during the git command to update the placeholder list have not yet completed at this point. @@ -1212,11 +1340,11 @@ private void UpdateOrDeletePlaceholder(KeyValuePair pathAndSha) this.placeholderList.Remove(virtualPath); break; - case StatusCode.StatusObjectNameNotFound: + case NtStatus.ObjectNameNotFound: this.placeholderList.Remove(virtualPath); break; - case StatusCode.StatusObjectPathNotFound: + case NtStatus.ObjectPathNotFound: this.placeholderList.Remove(virtualPath); break; @@ -1258,31 +1386,77 @@ private void LogErrorAndExit(string message, Exception e) Environment.Exit(1); } - private void CopyIndexFileAndParse() + private void CopyIndexFileAndBuildProjection() + { + this.context.FileSystem.CopyFile(this.indexPath, this.projectionIndexBackupPath, overwrite: true); + this.BuildProjection(); + } + + private void BuildProjection() { - this.context.FileSystem.CopyFile(this.indexPath, this.projectionIndexBackupPath, overwrite: true); this.SetProjectionInvalid(false); this.offsetsInvalid = false; - this.ParseProjection(); + + using (FileStream indexStream = new FileStream(this.projectionIndexBackupPath, FileMode.Open, FileAccess.Read, FileShare.Read, IndexFileStreamBufferSize)) + { + try + { + this.ParseIndexAndBuildProjection(indexStream); + } + catch (Exception e) + { + EventMetadata metadata = CreateEventMetadata("BuildProjection: Exception thrown by ParseIndexAndBuildProjection", e); + this.context.Tracer.RelatedEvent(EventLevel.Warning, "BuildProjection_ParseIndexAndBuildProjectionException", metadata); + + this.SetProjectionInvalid(true); + this.offsetsInvalid = true; + throw; + } + } } - private void ParseProjection() + private void ParseIndexAndBuildProjection(FileStream indexFileStream) { - using (FileStream indexStream = new FileStream(this.projectionIndexBackupPath, FileMode.Open, FileAccess.Read, FileShare.Read, IndexFileStreamBufferSize)) + CallbackResult result = this.TryIndexAction(indexFileStream, IndexAction.RebuildProjection); { - this.ParseIndex(indexStream, updateOffsetsOnly: false); + if (result != CallbackResult.Success) + { + throw new InvalidOperationException("ParseIndexAndBuildProjection: TryIndexAction failed to rebuild projection"); + } } } - private void ParseIndex(FileStream indexFileStream, bool updateOffsetsOnly) + /// + /// Takes an action using the index in indexFileStream + /// + /// FileStream for a git index file + /// Action to take using the specified index + /// + /// CallbackResult indicating success or failure of the specified action + /// + private CallbackResult TryIndexAction(FileStream indexFileStream, IndexAction action) { byte[] buffer = new byte[40]; indexFileStream.Position = 0; + + indexFileStream.Read(buffer, 0, 4); + if (buffer[0] != 'D' || + buffer[1] != 'I' || + buffer[2] != 'R' || + buffer[3] != 'C') + { + throw new InvalidDataException("Incorrect magic signature for index: " + string.Join(string.Empty, buffer.Take(4).Select(c => (char)c))); + } + + uint indexVersion = ReadUInt32(buffer, indexFileStream); + if (indexVersion != 4) + { + throw new InvalidDataException("Unsupported index version: " + indexVersion); + } - indexFileStream.Read(buffer, 0, 8); uint entryCount = ReadUInt32(buffer, indexFileStream); - if (!updateOffsetsOnly) + if (action == IndexAction.RebuildProjection) { this.projectionFolderCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); this.rootFolderData = new FileOrFolderData(); @@ -1296,20 +1470,13 @@ private void ParseIndex(FileStream indexFileStream, bool updateOffsetsOnly) byte[] sha = new byte[20]; for (int i = 0; i < entryCount; i++) { - // If the projection or offsets get set as invalid while being parsed we can bail - // since the index will have to be reparsed - if ((updateOffsetsOnly && this.offsetsInvalid) || (!updateOffsetsOnly && this.projectionInvalid)) - { - return; - } - long entryOffset = indexFileStream.Position; indexFileStream.Read(buffer, 0, 40); indexFileStream.Read(sha, 0, 20); ushort flags = ReadUInt16(buffer, indexFileStream); bool isExtended = (flags & ExtendedBit) == ExtendedBit; - int pathLength = (ushort)(((flags << 20) >> 20) & 4095); + int pathLength = (ushort)(flags & 0xFFF); bool skipWorktree = false; if (isExtended) @@ -1320,22 +1487,71 @@ private void ParseIndex(FileStream indexFileStream, bool updateOffsetsOnly) int replaceLength = ReadReplaceLength(indexFileStream); int replaceIndex = previousPathLength - replaceLength; - indexFileStream.Read(pathBuffer, replaceIndex, pathLength - replaceIndex + 1); + int value = pathLength - replaceIndex + 1; + indexFileStream.Read(pathBuffer, replaceIndex, value); previousPathLength = pathLength; - if (skipWorktree) + + switch (action) { - string path = Encoding.UTF8.GetString(pathBuffer, 0, pathLength); - if (updateOffsetsOnly) - { - this.UpdateFileOffset(path, entryOffset, ref lastParent, ref lastParentPath); - } - else - { - this.AddItem(path, sha, entryOffset, ref lastParent, ref lastParentPath); - } + case IndexAction.RebuildProjection: + if (skipWorktree) + { + string path = Encoding.UTF8.GetString(pathBuffer, 0, pathLength); + this.AddItem(path, sha, entryOffset, ref lastParent, ref lastParentPath); + } + + break; + + case IndexAction.UpdateOffsets: + if (skipWorktree) + { + string path = Encoding.UTF8.GetString(pathBuffer, 0, pathLength); + this.UpdateFileOffset(path, entryOffset, ref lastParent, ref lastParentPath); + } + + break; + + case IndexAction.ValidateSparseCheckout: + if (!skipWorktree) + { + // A git command (e.g. 'git reset --mixed') may have cleared a file's skip worktree bit without + // updating the sparse-checkout file. Ensure this file is in the sparse-checkout file + string path = Encoding.UTF8.GetString(pathBuffer, 0, pathLength); + CallbackResult updateSparseCheckoutResult = this.sparseCheckout.AddFileEntryFromIndex(path); + if (updateSparseCheckoutResult != CallbackResult.Success) + { + return updateSparseCheckoutResult; + } + } + + break; + + case IndexAction.UpdateOffsetsAndValidateSparseCheckout: + { + string path = Encoding.UTF8.GetString(pathBuffer, 0, pathLength); + if (skipWorktree) + { + this.UpdateFileOffset(path, entryOffset, ref lastParent, ref lastParentPath); + } + else + { + CallbackResult updateSparseCheckoutResult = this.sparseCheckout.AddFileEntryFromIndex(path); + if (updateSparseCheckoutResult != CallbackResult.Success) + { + return updateSparseCheckoutResult; + } + } + } + + break; + + default: + throw new ArgumentOutOfRangeException(nameof(action)); } } - } + + return CallbackResult.Success; + } // Wrapper for FileOrFolderData that allows for caching string SHAs private class FileMissingSize @@ -1624,7 +1840,7 @@ private static void BytesToCharArray(char[] shaString, int startIndex, ulong sha EventMetadata metadata = this.CreateErrorMetadata("PopulateMissingSizesFromRemote: Failed to download size for child entry"); metadata.Add("SHA", childNeedingSize.Sha); tracer.RelatedError(metadata, Keywords.Network); - throw new GvFltException(StatusCode.StatusFileNotAvailable); + throw new GvFltException(NtStatus.FileNotAvailable); } } diff --git a/GVFS/GVFS.GVFlt/DotGit/IProfilerOnlyIndexProjection.cs b/GVFS/GVFS.GVFlt/DotGit/IProfilerOnlyIndexProjection.cs index e8a8825958..718e88f4d1 100644 --- a/GVFS/GVFS.GVFlt/DotGit/IProfilerOnlyIndexProjection.cs +++ b/GVFS/GVFS.GVFlt/DotGit/IProfilerOnlyIndexProjection.cs @@ -7,8 +7,10 @@ /// public interface IProfilerOnlyIndexProjection { - void ForceParseIndexFileForNewProjection(); + void ForceRebuildProjection(); - void ForceParseIndexToUpdateOffsets(); + void ForceUpdateOffsetsAndValidateSparseCheckout(); + + void ForceValidateSparseCheckout(); } } diff --git a/GVFS/GVFS.GVFlt/DotGit/SparseCheckout.cs b/GVFS/GVFS.GVFlt/DotGit/SparseCheckout.cs index 7b5c336f8a..2da38dfcf0 100644 --- a/GVFS/GVFS.GVFlt/DotGit/SparseCheckout.cs +++ b/GVFS/GVFS.GVFlt/DotGit/SparseCheckout.cs @@ -1,8 +1,10 @@ using GVFS.Common; +using GVFS.Common.Git; using GVFS.Common.Tracing; using System; +using System.Collections; +using System.Collections.Generic; using System.IO; -using GVFS.Common.Git; namespace GVFS.GVFlt.DotGit { @@ -22,13 +24,15 @@ public SparseCheckout(GVFSContext context, string virtualSparseCheckoutFilePath) this.sparseCheckoutSerializer = new FileSerializer(context, virtualSparseCheckoutFilePath); this.context = context; } + + public IEnumerable Entries + { + get { return this.sparseCheckoutEntries; } + } public int EntryCount { - get - { - return this.sparseCheckoutEntries.Count; - } + get { return this.sparseCheckoutEntries.Count; } } public void LoadOrCreate() @@ -58,14 +62,30 @@ public bool HasEntry(string virtualPath, bool isFolder) public CallbackResult AddFileEntry(string virtualPath) { - return this.AddEntry(virtualPath, isFolder: false); + string entry = this.NormalizeEntryString(virtualPath, isFolder: false); + return this.AddEntry(entry, isFolder: false); } public CallbackResult AddFolderEntry(string virtualPath) { - return this.AddEntry(virtualPath, isFolder: true); + string entry = this.NormalizeEntryString(virtualPath, isFolder: true); + return this.AddEntry(entry, isFolder: true); } + public CallbackResult AddFileEntryFromIndex(string gitPath) + { + string entry = this.NormalizeEntryString(gitPath, isFolder: false); + + // Check Contains before calling AddEntry as Contains is lower weight than Add, and the vast + // majority of the time entries being added from the index will already be in the sparse-checkout file + if (!this.sparseCheckoutEntries.Contains(entry)) + { + return this.AddEntry(entry, isFolder: false); + } + + return CallbackResult.Success; + } + private string NormalizeEntryString(string virtualPath, bool isFolder) { return GVFSConstants.GitPathSeparatorString + @@ -73,28 +93,27 @@ private string NormalizeEntryString(string virtualPath, bool isFolder) (isFolder ? GVFSConstants.GitPathSeparatorString : string.Empty); } - private CallbackResult AddEntry(string virtualPath, bool isFolder) - { - string entry = this.NormalizeEntryString(virtualPath, isFolder); - if (this.sparseCheckoutEntries.Add(entry)) + private CallbackResult AddEntry(string normalizedEntry, bool isFolder) + { + if (this.sparseCheckoutEntries.Add(normalizedEntry)) { try { - this.sparseCheckoutSerializer.AppendLine(entry); + this.sparseCheckoutSerializer.AppendLine(normalizedEntry); } catch (IOException e) { CallbackResult result = CallbackResult.RetryableError; EventMetadata metadata = new EventMetadata(); metadata.Add("Area", "SparseCheckout"); - metadata.Add("virtualFolderPath", virtualPath); + metadata.Add("normalizedEntry", normalizedEntry); metadata.Add("isFolder", isFolder); metadata.Add("Exception", e.ToString()); metadata.Add("ErrorMessage", "IOException caught while processing AddEntry"); // Remove the entry so that if AddRecursiveSparseCheckoutEntry is called again // we'll try to append to the file again - if (!this.sparseCheckoutEntries.TryRemove(entry)) + if (!this.sparseCheckoutEntries.TryRemove(normalizedEntry)) { metadata["ErrorMessage"] += ", failed to undo addition to sparseCheckoutEntries"; result = CallbackResult.FatalError; @@ -107,7 +126,7 @@ private CallbackResult AddEntry(string virtualPath, bool isFolder) { EventMetadata metadata = new EventMetadata(); metadata.Add("Area", "SparseCheckout"); - metadata.Add("virtualFolderPath", virtualPath); + metadata.Add("normalizedEntry", normalizedEntry); metadata.Add("isFolder", isFolder); metadata.Add("Exception", e.ToString()); metadata.Add("ErrorMessage", "Exception caught while processing AddEntry"); diff --git a/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj b/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj index 230f9c676d..41d2a87646 100644 --- a/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj +++ b/GVFS/GVFS.GVFlt/GVFS.GVFlt.csproj @@ -98,9 +98,9 @@ {374bf1e5-0b2d-4d4a-bd5e-4212299def09} GVFS.Common - + {fb0831ae-9997-401b-b31f-3a065fdbeb20} - GVFS.GvFltWrapper + GvFlt diff --git a/GVFS/GVFS.GVFlt/GVFltCallbacks.cs b/GVFS/GVFS.GVFlt/GVFltCallbacks.cs index 3e5ef6b235..63dd3d9b70 100644 --- a/GVFS/GVFS.GVFlt/GVFltCallbacks.cs +++ b/GVFS/GVFS.GVFlt/GVFltCallbacks.cs @@ -1,11 +1,11 @@ -using GVFS.Common; +using GvFlt; +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Git; using GVFS.Common.NamedPipes; -using GVFS.Common.Physical; -using GVFS.Common.Physical.FileSystem; -using GVFS.Common.Physical.Git; +using GVFS.Common.NetworkStreams; using GVFS.Common.Tracing; using GVFS.GVFlt.DotGit; -using GVFSGvFltWrapper; using Microsoft.Database.Isam.Config; using Microsoft.Diagnostics.Tracing; using Microsoft.Isam.Esent.Collections.Generic; @@ -15,25 +15,28 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Threading; +using ITracer = GVFS.Common.Tracing.ITracer; + namespace GVFS.GVFlt { public class GVFltCallbacks : IDisposable, IHeartBeatMetadataProvider { + public const byte PlaceholderVersion = 1; + private const int MaxBlobStreamBufferSize = 64 * 1024; private const string RefMarker = "ref:"; private const string EtwArea = "GVFltCallbacks"; private const int BlockSize = 64 * 1024; - private const int AcquireGVFSLockRetries = 50; - private const int AcquireGVFSLockWaitPerTryMillis = 600; private const int MinGvFltThreads = 5; private static readonly string RefsHeadsPath = GVFSConstants.DotGit.Refs.Heads.Root + GVFSConstants.PathSeparator; private readonly string logsHeadPath; - private GvFltWrapper gvflt; + private VirtualizationInstance gvflt; private object stopLock = new object(); private bool gvfltIsStarted = false; private bool isMountComplete = false; @@ -56,7 +59,7 @@ public GVFltCallbacks(GVFSContext context, GVFSGitObjects gitObjects, RepoMetada this.context = context; this.repoMetadata = repoMetadata; this.logsHeadFileProperties = null; - this.gvflt = new GvFltWrapper(); + this.gvflt = new VirtualizationInstance(); this.activeEnumerations = new ConcurrentDictionary(); this.sparseCheckout = new SparseCheckout( this.context, @@ -70,7 +73,14 @@ public GVFltCallbacks(GVFSContext context, GVFSGitObjects gitObjects, RepoMetada }); this.gvfsGitObjects = gitObjects; - this.gitIndexProjection = new GitIndexProjection(context, gitObjects, this.blobSizes, this.repoMetadata, this.gvflt); + this.gitIndexProjection = new GitIndexProjection( + context, + gitObjects, + this.blobSizes, + this.repoMetadata, + this.gvflt, + new PersistentDictionary(Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DatabaseNames.PlaceholderList)), + this.sparseCheckout); this.background = new ReliableBackgroundOperations( this.context, @@ -92,7 +102,7 @@ public static bool TryPrepareFolderForGVFltCallbacks(string folderPath, out stri { error = string.Empty; Guid virtualizationInstanceGuid = Guid.NewGuid(); - HResult result = GvFltWrapper.GvConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); if (result != HResult.Ok) { error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); @@ -123,6 +133,26 @@ public static bool IsPathMonitoredForWrites(string virtualPath) return false; } + public static string GetShaFromContentId(byte[] contentId) + { + return Encoding.Unicode.GetString(contentId, 0, GVFSConstants.ShaStringLength * sizeof(char)); + } + + public static byte GetPlaceholderVersionFromEpochId(byte[] epochId) + { + return epochId[0]; + } + + public static byte[] ConvertShaToContentId(string sha) + { + return Encoding.Unicode.GetBytes(sha); + } + + public static byte[] GetEpochId() + { + return new byte[] { PlaceholderVersion }; + } + public NamedPipeMessages.ReleaseLock.Response TryReleaseExternalLock(int pid) { return this.gitIndexProjection.TryReleaseExternalLock(pid); @@ -150,11 +180,11 @@ public bool TryStart(out string error) this.gvflt.OnEndDirectoryEnumeration = this.GVFltEndDirectoryEnumerationHandler; this.gvflt.OnGetDirectoryEnumeration = this.GVFltGetDirectoryEnumerationHandler; this.gvflt.OnQueryFileName = this.GVFltQueryFileNameHandler; - this.gvflt.OnGetPlaceHolderInformation = this.GVFltGetPlaceHolderInformationHandler; + this.gvflt.OnGetPlaceholderInformation = this.GVFltGetPlaceholderInformationHandler; this.gvflt.OnGetFileStream = this.GVFltGetFileStreamHandler; this.gvflt.OnNotifyFirstWrite = this.GVFltNotifyFirstWriteHandler; - this.gvflt.OnNotifyCreate = this.GVFltNotifyCreateHandler; + this.gvflt.OnNotifyFileHandleCreated = this.GVFltNotifyFileHandleCreatedHandler; this.gvflt.OnNotifyPreDelete = this.GVFltNotifyPreDeleteHandler; this.gvflt.OnNotifyPreRename = this.GvFltNotifyPreRenameHandler; this.gvflt.OnNotifyPreSetHardlink = null; @@ -166,8 +196,8 @@ public bool TryStart(out string error) // We currently use twice as many threads as connections to allow for // non-network operations to possibly succeed despite the connection limit - HResult result = this.gvflt.GvStartVirtualizationInstance( - this.context.Tracer, + HResult result = this.gvflt.StartVirtualizationInstance( + new GvFltTracer(this.context.Tracer), this.context.Enlistment.WorkingDirectoryRoot, poolThreadCount: threadCount, concurrentThreadCount: threadCount); @@ -202,8 +232,8 @@ public void Stop() if (this.gvfltIsStarted) { - this.gvflt.GvStopVirtualizationInstance(); - this.gvflt.GvDetachDriver(); + this.gvflt.StopVirtualizationInstance(); + this.gvflt.DetachDriver(); Console.WriteLine("GVFlt callbacks stopped"); this.gvfltIsStarted = false; } @@ -283,99 +313,59 @@ protected virtual void Dispose(bool disposing) } } - private static EventMetadata CreatePathEventMetadata(string area, string relativeFilePath) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", area); - metadata.Add("relativeFilePath", relativeFilePath); - - return metadata; - } - private void OnIndexFileChange() { string lockedGitCommand = this.context.Repository.GVFSLock.GetLockedGitCommand(); - if (string.IsNullOrEmpty(lockedGitCommand)) + GitCommandLineParser gitCommand = new GitCommandLineParser(lockedGitCommand); + if (this.gitIndexProjection.IsIndexBeingUpdatedByGVFS()) { - if (!this.gitIndexProjection.IsIndexBeingUpdatedByGVFS()) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", EtwArea); - metadata.Add("Message", "Index modified without git holding GVFS lock"); - this.context.Tracer.RelatedEvent(EventLevel.Warning, "OnIndexFileChange", metadata); + // No need to invalidate anything, because this event came from our own background thread writing to the index - // TODO 935249: Investigate if index should have its offsets or projection invalidated - // if the GVFS lock is not held when the index is written to - this.gitIndexProjection.InvalidateOffsets(); - } - } - else if (this.GitCommandLeavesProjectionUnchanged(lockedGitCommand)) - { - bool canSkipInvalidation = GitHelper.IsVerb(lockedGitCommand, "status") && lockedGitCommand.Contains("--no-lock-index"); - if (!canSkipInvalidation) + if (gitCommand.IsValidGitCommand) { - this.gitIndexProjection.InvalidateOffsets(); + // But there should never be a case where GVFS is writing to the index while Git is holding the lock + EventMetadata metadata = new EventMetadata + { + { "Area", EtwArea }, + { "Message", "GVFS wrote to the index while git was holding the GVFS lock" }, + { "GitCommand", lockedGitCommand }, + }; + + this.context.Tracer.RelatedEvent(EventLevel.Warning, "OnIndexFileChange_LockCollision", metadata); } } - else + else if (!gitCommand.IsValidGitCommand) { + // Something wrote to the index without holding the GVFS lock, so we invalidate the projection this.gitIndexProjection.InvalidateProjection(); - } - } - - private bool GitCommandLeavesProjectionUnchanged(string lockedGitCommand) - { - if (GitHelper.IsVerb(lockedGitCommand, "add") || - GitHelper.IsVerb(lockedGitCommand, "branch") || - GitHelper.IsVerb(lockedGitCommand, "commit") || - GitHelper.IsVerb(lockedGitCommand, "status") || - GitHelper.IsVerb(lockedGitCommand, "update-index") || - GitHelper.IsVerb(lockedGitCommand, "update-ref") || - this.GitCommandIsResetLeavingProjectionUnchanged(lockedGitCommand)) - { - return true; - } - - return false; - } - private bool GitCommandIsResetLeavingProjectionUnchanged(string gitCommand) - { - // TODO 940173: Be more robust when parsing git arguments - if (!GitHelper.IsVerb(gitCommand, "reset")) - { - return false; - } + // But this isn't something we expect to see, so log a warning + EventMetadata metadata = new EventMetadata + { + { "Area", EtwArea }, + { "Message", "Index modified without git holding GVFS lock" }, + }; - if (gitCommand.Contains(" --hard ") || gitCommand.EndsWith(" --hard")) - { - return false; + this.context.Tracer.RelatedEvent(EventLevel.Warning, "OnIndexFileChange_NoLock", metadata); } - - if (gitCommand.Contains(" --keep ") || gitCommand.EndsWith(" --keep")) + else if (this.GitCommandLeavesProjectionUnchanged(gitCommand)) { - return false; + this.gitIndexProjection.InvalidateOffsetsAndSparseCheckout(); + this.background.Enqueue(BackgroundGitUpdate.OnIndexWriteWithoutProjectionChange()); } - - if (gitCommand.Contains(" --merge ") || gitCommand.EndsWith(" --merge")) + else { - return false; + this.gitIndexProjection.InvalidateProjection(); } - - return true; } - private bool GitCommandIsResetHard(string gitCommand) + private bool GitCommandLeavesProjectionUnchanged(GitCommandLineParser gitCommand) { - // TODO 940173: Be more robust when parsing git arguments - if (GitHelper.IsVerb(gitCommand, "reset") && - (gitCommand.Contains(" --hard ") || gitCommand.EndsWith(" --hard"))) - { - return true; - } - - return false; - } + return + gitCommand.IsVerb("add", "commit", "status", "update-index") || + gitCommand.IsResetSoftOrMixed() || + gitCommand.IsCheckoutWithFilePaths(); + } private void OnLogsHeadChange() { @@ -383,7 +373,7 @@ private void OnLogsHeadChange() this.logsHeadFileProperties = null; } - private StatusCode GVFltStartDirectoryEnumerationHandler(Guid enumerationId, string virtualPath) + private NtStatus GVFltStartDirectoryEnumerationHandler(Guid enumerationId, string virtualPath) { virtualPath = PathUtil.RemoveTrailingSlashIfPresent(virtualPath); @@ -395,7 +385,7 @@ private StatusCode GVFltStartDirectoryEnumerationHandler(Guid enumerationId, str metadata.Add("enumerationId", enumerationId); this.context.Tracer.RelatedEvent(EventLevel.Informational, "StartDirectoryEnum_MountNotComplete", metadata); - return StatusCode.StatusDeviceNotReady; + return NtStatus.DeviceNotReady; } GVFltActiveEnumeration activeEnumeration = new GVFltActiveEnumeration(this.gitIndexProjection.GetProjectedItems(virtualPath)); @@ -410,13 +400,13 @@ private StatusCode GVFltStartDirectoryEnumerationHandler(Guid enumerationId, str this.context.Tracer.RelatedError(metadata); activeEnumeration.Dispose(); - return StatusCode.StatusInvalidParameter; + return NtStatus.InvalidParameter; } - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } - private StatusCode GVFltEndDirectoryEnumerationHandler(Guid enumerationId) + private NtStatus GVFltEndDirectoryEnumerationHandler(Guid enumerationId) { GVFltActiveEnumeration activeEnumeration; if (this.activeEnumerations.TryRemove(enumerationId, out activeEnumeration)) @@ -433,17 +423,17 @@ private StatusCode GVFltEndDirectoryEnumerationHandler(Guid enumerationId) metadata.Add("enumerationId", enumerationId); this.context.Tracer.RelatedError(metadata); - return StatusCode.StatusInvalidParameter; + return NtStatus.InvalidParameter; } - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } - private StatusCode GVFltGetDirectoryEnumerationHandler( + private NtStatus GVFltGetDirectoryEnumerationHandler( Guid enumerationId, string filterFileName, bool restartScan, - GvDirectoryEnumerationResult result) + DirectoryEnumerationResult result) { GVFltActiveEnumeration activeEnumeration = null; if (!this.activeEnumerations.TryGetValue(enumerationId, out activeEnumeration)) @@ -458,7 +448,7 @@ private StatusCode GVFltEndDirectoryEnumerationHandler(Guid enumerationId) metadata.Add("restartScan", restartScan); this.context.Tracer.RelatedError(metadata); - return StatusCode.StatusInternalError; + return NtStatus.InternalError; } bool initialRequest; @@ -495,32 +485,32 @@ private StatusCode GVFltEndDirectoryEnumerationHandler(Guid enumerationId) if (result.TrySetFileName(fileInfo.Name)) { - // Only advance the enumeration if the file name fit in the GvDirectoryEnumerationResult + // Only advance the enumeration if the file name fit in the DirectoryEnumerationResult activeEnumeration.MoveNext(); - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } else { // Return StatusBufferOverflow to indicate that the file name had to be truncated - return StatusCode.StatusBufferOverflow; + return NtStatus.BufferOverflow; } } // TODO 636568: Confirm return code values/behavior with GVFlt team - StatusCode statusCode = (initialRequest && PathUtil.IsEnumerationFilterSet(filterFileName)) ? StatusCode.StatusNoSuchFile : StatusCode.StatusNoMoreFiles; + NtStatus statusCode = (initialRequest && PathUtil.IsEnumerationFilterSet(filterFileName)) ? NtStatus.NoSuchFile : NtStatus.NoMoreFiles; return statusCode; } /// - /// GVFltQueryFileNameHandler is called by GVFlt when a file is being deleted or renamed. It is an optimiation so that GVFlt + /// GVFltQueryFileNameHandler is called by GVFlt when a file is being deleted or renamed. It is an optimization so that GVFlt /// can avoid calling Start\Get\End enumeration to check if GVFS is still projecting a file. This method uses the same /// rules for deciding what is projected as the enumeration callbacks. /// - private StatusCode GVFltQueryFileNameHandler(string virtualPath) + private NtStatus GVFltQueryFileNameHandler(string virtualPath) { if (PathUtil.IsPathInsideDotGit(virtualPath)) { - return StatusCode.StatusObjectNameNotFound; + return NtStatus.ObjectNameNotFound; } virtualPath = PathUtil.RemoveTrailingSlashIfPresent(virtualPath); @@ -529,19 +519,19 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) { EventMetadata metadata = this.CreateEventMetadata("GVFltQueryFileNameHandler: Mount has not yet completed", virtualPath); this.context.Tracer.RelatedEvent(EventLevel.Informational, "QueryFileName_MountNotComplete", metadata); - return StatusCode.StatusDeviceNotReady; + return NtStatus.DeviceNotReady; } bool isFolder; if (!this.gitIndexProjection.IsPathProjected(virtualPath, out isFolder)) { - return StatusCode.StatusObjectNameNotFound; + return NtStatus.ObjectNameNotFound; } - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } - private StatusCode GVFltGetPlaceHolderInformationHandler( + private NtStatus GVFltGetPlaceholderInformationHandler( string virtualPath, uint desiredAccess, uint shareMode, @@ -554,7 +544,7 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) if (!this.isMountComplete) { - EventMetadata metadata = this.CreateEventMetadata("GVFltGetPlaceHolderInformationHandler: Mount has not yet completed", virtualPath); + EventMetadata metadata = this.CreateEventMetadata("GVFltGetPlaceholderInformationHandler: Mount has not yet completed", virtualPath); metadata.Add("desiredAccess", desiredAccess); metadata.Add("shareMode", shareMode); metadata.Add("createDisposition", createDisposition); @@ -563,38 +553,39 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) metadata.Add("triggeringProcessImageFileName", triggeringProcessImageFileName); this.context.Tracer.RelatedEvent(EventLevel.Informational, "GetPlaceHolder_MountNotComplete", metadata); - return StatusCode.StatusDeviceNotReady; + return NtStatus.DeviceNotReady; } string sha; - GVFltFileInfo fileInfo = this.gitIndexProjection.GetProjectedGVFltFileInfoAndSha(virtualPath, out sha); + string parentFolderPath; + GVFltFileInfo fileInfo = this.gitIndexProjection.GetProjectedGVFltFileInfoAndSha(virtualPath, out parentFolderPath, out sha); if (fileInfo == null) { - return StatusCode.StatusObjectNameNotFound; + return NtStatus.ObjectNameNotFound; } if (!fileInfo.IsFolder && !this.IsSpecialGitFile(fileInfo) && !this.CanCreatePlaceholder()) { - EventMetadata metadata = this.CreateEventMetadata("GVFltGetPlaceHolderInformationHandler: Not allowed to create placeholder", virtualPath); + EventMetadata metadata = this.CreateEventMetadata("GVFltGetPlaceholderInformationHandler: Not allowed to create placeholder", virtualPath); metadata.Add("desiredAccess", desiredAccess); metadata.Add("shareMode", shareMode); metadata.Add("createDisposition", createDisposition); metadata.Add("createOptions", createOptions); metadata.Add("triggeringProcessId", triggeringProcessId); metadata.Add("triggeringProcessImageFileName", triggeringProcessImageFileName); - this.context.Tracer.RelatedEvent(EventLevel.Verbose, nameof(this.GVFltGetPlaceHolderInformationHandler), metadata); + this.context.Tracer.RelatedEvent(EventLevel.Verbose, nameof(this.GVFltGetPlaceholderInformationHandler), metadata); // Another process is modifying the working directory so we cannot modify it // until they are done. - return StatusCode.StatusObjectNameNotFound; + return NtStatus.ObjectNameNotFound; } // The file name case in the virtualPath parameter might be different than the file name case in the repo. // Build a new virtualPath that preserves the case in the repo so that the placeholder file is created // with proper case. - string gitCaseVirtualPath = Path.Combine(Path.GetDirectoryName(virtualPath), fileInfo.Name); + string gitCaseVirtualPath = Path.Combine(parentFolderPath, fileInfo.Name); uint fileAttributes; if (fileInfo.IsFolder) @@ -607,7 +598,7 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) } FileProperties properties = this.GetLogsHeadFileProperties(); - StatusCode result = this.gvflt.GvWritePlaceholderInformation( + NtStatus result = this.gvflt.WritePlaceholderInformation( gitCaseVirtualPath, properties.CreationTimeUTC, properties.LastAccessTimeUTC, @@ -616,12 +607,12 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) fileAttributes: fileAttributes, endOfFile: fileInfo.Size, directory: fileInfo.IsFolder, - contentId: sha, - epochId: null); + contentId: ConvertShaToContentId(sha), + epochId: GVFltCallbacks.GetEpochId()); - if (result != StatusCode.StatusSucccess) + if (result != NtStatus.Succcess) { - EventMetadata metadata = this.CreateEventMetadata("GVFltGetPlaceHolderInformationHandler: GvWritePlaceholderInformation failed", virtualPath, exception: null, errorMessage: true); + EventMetadata metadata = this.CreateEventMetadata("GVFltGetPlaceholderInformationHandler: GvWritePlaceholderInformation failed", virtualPath, exception: null, errorMessage: true); metadata.Add("gitCaseVirtualPath", gitCaseVirtualPath); metadata.Add("desiredAccess", desiredAccess); metadata.Add("shareMode", shareMode); @@ -631,7 +622,7 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) metadata.Add("triggeringProcessImageFileName", triggeringProcessImageFileName); metadata.Add("FileName", fileInfo.Name); metadata.Add("IsFolder", fileInfo.IsFolder); - metadata.Add("StatusCode", result.ToString("X") + "(" + result.ToString("G") + ")"); + metadata.Add("NtStatus", result.ToString("X") + "(" + result.ToString("G") + ")"); this.context.Tracer.RelatedError(metadata); } else @@ -640,12 +631,12 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) { this.gitIndexProjection.OnPlaceholderFileCreated(gitCaseVirtualPath, sha); - // Note: Because GetPlaceHolderInformationHandler is not synchronized it is possible that GVFS will double count + // Note: Because GVFltGetPlaceholderInformationHandler is not synchronized it is possible that GVFS will double count // the creation of file placeholders if multiple requests for the same file are received at the same time on different // threads. this.placeHolderCreationCount.AddOrUpdate( triggeringProcessImageFileName, - new PlaceHolderCreateCounter(), + (imageName) => { return new PlaceHolderCreateCounter(); }, (key, oldCount) => { oldCount.Increment(); return oldCount; }); } } @@ -653,17 +644,18 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) return result; } - private StatusCode GVFltGetFileStreamHandler( + private NtStatus GVFltGetFileStreamHandler( string virtualPath, long byteOffset, uint length, Guid streamGuid, - string contentId, + byte[] contentId, + byte[] epochId, uint triggeringProcessId, - string triggeringProcessImageFileName, - GVFltWriteBuffer targetBuffer) + string triggeringProcessImageFileName) { - string sha = contentId; + string sha = GetShaFromContentId(contentId); + byte placeholderVersion = GetPlaceholderVersionFromEpochId(epochId); EventMetadata metadata = new EventMetadata(); metadata.Add("originalVirtualPath", virtualPath); @@ -673,100 +665,126 @@ private StatusCode GVFltQueryFileNameHandler(string virtualPath) metadata.Add("triggeringProcessId", triggeringProcessId); metadata.Add("triggeringProcessImageFileName", triggeringProcessImageFileName); metadata.Add("sha", sha); + metadata.Add("placeholderVersion", placeholderVersion); using (ITracer activity = this.context.Tracer.StartActivity("GetFileStream", EventLevel.Verbose, Keywords.Telemetry, metadata)) { if (!this.isMountComplete) { metadata.Add("Message", "GVFltGetFileStreamHandler failed, mount has not yet completed"); activity.RelatedEvent(EventLevel.Informational, "GetFileStream_MountNotComplete", metadata); - return StatusCode.StatusDeviceNotReady; + return NtStatus.DeviceNotReady; } if (byteOffset != 0) { metadata.Add("ErrorMessage", "Invalid Parameter: byteOffset must be 0"); activity.RelatedError(metadata); - return StatusCode.StatusInvalidParameter; + return NtStatus.InvalidParameter; } - if (!this.gvfsGitObjects.TryCopyBlobContentStream( - sha, - (stream, blobLength) => + if (placeholderVersion != PlaceholderVersion) { - if (blobLength != length) - { - metadata.Add("blobLength", blobLength); - metadata.Add("ErrorMessage", "Actual file length (blobLength) does not match requested length"); - activity.RelatedError(metadata); - - throw new GvFltException(StatusCode.StatusInvalidParameter); - } + metadata.Add("ErrorMessage", "GVFltGetFileStreamHandler: Unexpected placeholder version"); + activity.RelatedError(metadata); + return NtStatus.InternalError; + } - byte[] buffer = new byte[Math.Min(MaxBlobStreamBufferSize, blobLength)]; - long remainingData = blobLength; - while (remainingData > 0) + try + { + if (!this.gvfsGitObjects.TryCopyBlobContentStream( + sha, + (stream, blobLength) => { - uint bytesToCopy = (uint)Math.Min(remainingData, targetBuffer.Length); - - try - { - targetBuffer.Stream.Seek(0, SeekOrigin.Begin); - stream.CopyBlockTo(targetBuffer.Stream, bytesToCopy, buffer); - } - catch (IOException e) + if (blobLength != length) { - metadata.Add("Exception", e.ToString()); - metadata.Add("ErrorMessage", "IOException while copying to unmanaged buffer."); + metadata.Add("blobLength", blobLength); + metadata.Add("ErrorMessage", "Actual file length (blobLength) does not match requested length"); activity.RelatedError(metadata); - throw new GvFltException("IOException while copying to unmanaged buffer: " + e.Message, StatusCode.StatusFileNotAvailable); - } - long writeOffset = length - remainingData; + throw new GvFltException(NtStatus.InvalidParameter); + } - StatusCode writeResult = this.gvflt.GvWriteFile(streamGuid, targetBuffer, (ulong)writeOffset, bytesToCopy); - remainingData -= bytesToCopy; + byte[] buffer = new byte[Math.Min(MaxBlobStreamBufferSize, blobLength)]; + long remainingData = blobLength; - if (writeResult != StatusCode.StatusSucccess) + using (WriteBuffer targetBuffer = gvflt.CreateWriteBuffer()) { - switch (writeResult) + while (remainingData > 0) { - case StatusCode.StatusFileClosed: - // StatusFileClosed is expected, and occurs when an application closes a file handle before OnGetFileStream - // is complete - break; - - case StatusCode.StatusObjectNameNotFound: - // GvWriteFile may return STATUS_OBJECT_NAME_NOT_FOUND if the stream guid provided is not valid (doesn’t exist in the stream table). - // For each file expansion, GVFlt creates a new get stream session with a new stream guid, the session starts at the beginning of the - // file expansion, and ends after the GetFileStream command returns or times out. - // - // If we hit this in GVFS, the most common explanation is that we're calling GvWriteFile after the GVFlt thread waiting on the respose - // from GetFileStream has already timed out - metadata.Add("Message", "GvWriteFile returned StatusObjectNameNotFound"); - activity.RelatedEvent(EventLevel.Informational, "GetFileStream_ObjectNameNotFound", metadata); - break; + uint bytesToCopy = (uint)Math.Min(remainingData, targetBuffer.Length); - default: - metadata.Add("ErrorMessage", "GvWriteFile failed, error: " + writeResult.ToString("X") + "(" + writeResult.ToString("G") + ")"); + try + { + targetBuffer.Stream.Seek(0, SeekOrigin.Begin); + stream.CopyBlockTo(targetBuffer.Stream, bytesToCopy, buffer); + } + catch (IOException e) + { + metadata.Add("Exception", e.ToString()); + metadata.Add("ErrorMessage", "IOException while copying to unmanaged buffer."); activity.RelatedError(metadata); - break; - } + throw new GvFltException("IOException while copying to unmanaged buffer: " + e.Message, NtStatus.FileNotAvailable); + } + + long writeOffset = length - remainingData; + + NtStatus writeResult = this.gvflt.WriteFile(streamGuid, targetBuffer, (ulong)writeOffset, bytesToCopy); + remainingData -= bytesToCopy; + + if (writeResult != NtStatus.Succcess) + { + switch (writeResult) + { + case NtStatus.FileClosed: + // StatusFileClosed is expected, and occurs when an application closes a file handle before OnGetFileStream + // is complete + break; + + case NtStatus.ObjectNameNotFound: + // GvWriteFile may return STATUS_OBJECT_NAME_NOT_FOUND if the stream guid provided is not valid (doesn’t exist in the stream table). + // For each file expansion, GVFlt creates a new get stream session with a new stream guid, the session starts at the beginning of the + // file expansion, and ends after the GetFileStream command returns or times out. + // + // If we hit this in GVFS, the most common explanation is that we're calling GvWriteFile after the GVFlt thread waiting on the respose + // from GetFileStream has already timed out + metadata.Add("Message", "GvWriteFile returned StatusObjectNameNotFound"); + activity.RelatedEvent(EventLevel.Informational, "GetFileStream_ObjectNameNotFound", metadata); + break; + + default: + metadata.Add("ErrorMessage", "GvWriteFile failed, error: " + writeResult.ToString("X") + "(" + writeResult.ToString("G") + ")"); + activity.RelatedError(metadata); + break; + } - throw new GvFltException(writeResult); + throw new GvFltException(writeResult); + } + } } + })) + { + metadata.Add("ErrorMessage", "TryCopyBlobContentStream failed"); + activity.RelatedError(metadata); + return NtStatus.FileNotAvailable; } - })) + } + catch (GvFltException) + { + throw; + } + catch (Exception ex) { metadata.Add("ErrorMessage", "TryCopyBlobContentStream failed"); + metadata.Add("Exception", ex.ToString()); activity.RelatedError(metadata); - return StatusCode.StatusFileNotAvailable; + return NtStatus.FileNotAvailable; } - - return StatusCode.StatusSucccess; + + return NtStatus.Succcess; } } - private StatusCode GVFltNotifyFirstWriteHandler(string virtualPath) + private NtStatus GVFltNotifyFirstWriteHandler(string virtualPath) { virtualPath = PathUtil.RemoveTrailingSlashIfPresent(virtualPath); @@ -774,7 +792,7 @@ private StatusCode GVFltNotifyFirstWriteHandler(string virtualPath) { EventMetadata metadata = this.CreateEventMetadata("GVFltNotifyFirstWriteHandler: Mount has not yet completed", virtualPath); this.context.Tracer.RelatedEvent(EventLevel.Informational, "NotifyFirstWrite_MountNotComplete", metadata); - return StatusCode.StatusDeviceNotReady; + return NtStatus.DeviceNotReady; } if (string.Equals(virtualPath, string.Empty)) @@ -799,10 +817,10 @@ private StatusCode GVFltNotifyFirstWriteHandler(string virtualPath) } } - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } - private void GVFltNotifyCreateHandler( + private void GVFltNotifyFileHandleCreatedHandler( string virtualPath, bool isDirectory, uint desiredAccess, @@ -856,7 +874,7 @@ private StatusCode GVFltNotifyFirstWriteHandler(string virtualPath) } } - private StatusCode GvFltNotifyPreRenameHandler(string relativePath, string destinationPath) + private NtStatus GvFltNotifyPreRenameHandler(string relativePath, string destinationPath) { if (destinationPath.Equals(GVFSConstants.DotGit.Index, StringComparison.OrdinalIgnoreCase)) { @@ -868,21 +886,21 @@ private StatusCode GvFltNotifyPreRenameHandler(string relativePath, string desti metadata.Add("Message", "Blocked index rename outside the lock"); this.context.Tracer.RelatedEvent(EventLevel.Warning, "GvFltNotifyPreRenameHandler", metadata); - return StatusCode.StatusAccessDenied; + return NtStatus.AccessDenied; } } - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } - private StatusCode GVFltNotifyPreDeleteHandler(string virtualPath, bool isDirectory) + private NtStatus GVFltNotifyPreDeleteHandler(string virtualPath, bool isDirectory) { if (PathUtil.IsPathInsideDotGit(virtualPath)) { virtualPath = PathUtil.RemoveTrailingSlashIfPresent(virtualPath); if (!DoesPathAllowDelete(virtualPath)) { - return StatusCode.StatusAccessDenied; + return NtStatus.AccessDenied; } } else if (isDirectory) @@ -895,20 +913,20 @@ private StatusCode GVFltNotifyPreDeleteHandler(string virtualPath, bool isDirect // Respond with something that Git expects, StatusAccessDenied will lock up Git. // The directory is not exactly not-empty but it’s potentially not-empty // within the timeline of the current git command which is the reason for us blocking the delete. - return StatusCode.StatusDirectoryNotEmpty; + return NtStatus.DirectoryNotEmpty; } } - return StatusCode.StatusSucccess; + return NtStatus.Succcess; } private bool CanDeleteDirectory() { - string lockedGitCommand = this.context.Repository.GVFSLock.GetLockedGitCommand(); + GitCommandLineParser gitCommand = new GitCommandLineParser(this.context.Repository.GVFSLock.GetLockedGitCommand()); return - string.IsNullOrEmpty(lockedGitCommand) || - GitHelper.IsVerb(lockedGitCommand, "clean") || - this.GitCommandIsResetHard(lockedGitCommand); + !gitCommand.IsValidGitCommand || + gitCommand.IsVerb("clean") || + gitCommand.IsResetHard(); } private void GVFltNotifyFileRenamedHandler( @@ -981,21 +999,21 @@ private void OnDotGitFileChanged(string virtualPath) private uint GetDotGitNotificationMask(string virtualPath) { - uint notificationMask = (uint)GvNotificationType.NotificationFileRenamed; + uint notificationMask = (uint)NotificationType.FileRenamed; if (!DoesPathAllowDelete(virtualPath)) { - notificationMask |= (uint)GvNotificationType.NotificationPreDelete; + notificationMask |= (uint)NotificationType.PreDelete; } if (IsPathMonitoredForWrites(virtualPath)) { - notificationMask |= (uint)GvNotificationType.NotificationFileHandleClosed; + notificationMask |= (uint)NotificationType.FileHandleClosed; } if (virtualPath.Equals(GVFSConstants.DotGit.IndexLock, StringComparison.OrdinalIgnoreCase)) { - notificationMask |= (uint)GvNotificationType.NotificationPreRename; + notificationMask |= (uint)NotificationType.PreRename; } return notificationMask; @@ -1003,14 +1021,14 @@ private uint GetDotGitNotificationMask(string virtualPath) private uint GetWorkingDirectoryNotificationMask(bool isDirectory) { - uint notificationMask = (uint)GvNotificationType.NotificationFileRenamed; + uint notificationMask = (uint)NotificationType.FileRenamed; if (isDirectory) { - notificationMask |= (uint)GvNotificationType.NotificationPreDelete; + notificationMask |= (uint)NotificationType.PreDelete; } - notificationMask |= (uint)GvNotificationType.NotificationFileHandleClosed; + notificationMask |= (uint)NotificationType.FileHandleClosed; return notificationMask; } @@ -1159,6 +1177,10 @@ private CallbackResult ExecuteBackgroundOperation(BackgroundGitUpdate gitUpdate) result = this.alwaysExcludeFile.AddEntriesForFileOrFolder(gitUpdate.VirtualPath, isFolder: true); break; + case BackgroundGitUpdate.OperationType.OnIndexWriteWithoutProjectionChange: + result = this.gitIndexProjection.ValidateSparseCheckout(); + break; + default: throw new InvalidOperationException("Invalid background operation"); } @@ -1246,18 +1268,6 @@ private bool IsSpecialGitFile(GVFltFileInfo fileInfo) return metadata; } - private int GetMaxGVFSLockAttempts() - { - if (this.context.Repository.GVFSLock.IsLockedByGitVerb("commit")) - { - return AcquireGVFSLockRetries; - } - else - { - return 1; - } - } - private FileProperties GetLogsHeadFileProperties() { // Use a temporary FileProperties in case another thread sets this.logsHeadFileProperties before this @@ -1297,8 +1307,10 @@ private FileProperties GetLogsHeadFileProperties() /// private bool CanCreatePlaceholder() { - string lockedCommand = this.context.Repository.GVFSLock.GetLockedGitCommand(); - return string.IsNullOrEmpty(lockedCommand) || GitHelper.IsVerb(lockedCommand, "status", "add", "mv"); + GitCommandLineParser gitCommand = new GitCommandLineParser(this.context.Repository.GVFSLock.GetLockedGitCommand()); + return + !gitCommand.IsValidGitCommand || + gitCommand.IsVerb("status", "add", "mv"); } [Serializable] @@ -1328,6 +1340,7 @@ public enum OperationType OnFolderRenamed, OnFolderDeleted, OnFolderFirstWrite, + OnIndexWriteWithoutProjectionChange, } public OperationType Operation { get; set; } @@ -1397,6 +1410,11 @@ public static BackgroundGitUpdate OnFolderFirstWrite(string virtualPath) return new BackgroundGitUpdate(OperationType.OnFolderFirstWrite, virtualPath, oldVirtualPath: null); } + public static BackgroundGitUpdate OnIndexWriteWithoutProjectionChange() + { + return new BackgroundGitUpdate(OperationType.OnIndexWriteWithoutProjectionChange, virtualPath: null, oldVirtualPath: null); + } + public override string ToString() { return JsonConvert.SerializeObject(this); @@ -1406,6 +1424,7 @@ public override string ToString() private class PlaceHolderCreateCounter { private long count; + public PlaceHolderCreateCounter() { this.count = 1; @@ -1421,5 +1440,30 @@ public void Increment() Interlocked.Increment(ref this.count); } } + + private class GvFltTracer : GvFlt.ITracer + { + private ITracer tracer; + + public GvFltTracer(ITracer tracer) + { + this.tracer = tracer; + } + + public void TraceError(string message) + { + this.tracer.RelatedError(message); + } + + public void TraceError(Dictionary metadata) + { + this.tracer.RelatedError(new EventMetadata(metadata)); + } + + public void TraceEvent(EventLevel level, string eventName, Dictionary metadata) + { + this.tracer.RelatedEvent(level, eventName, new EventMetadata(metadata)); + } + } } } diff --git a/GVFS/GVFS.GVFlt/PathUtil.cs b/GVFS/GVFS.GVFlt/PathUtil.cs index 4e9cc0b7cc..8c3fd3c29b 100644 --- a/GVFS/GVFS.GVFlt/PathUtil.cs +++ b/GVFS/GVFS.GVFlt/PathUtil.cs @@ -1,6 +1,5 @@ using GVFS.Common; using System; -using System.Linq; namespace GVFS.GVFlt { diff --git a/GVFS/GVFS.GvFltWrapper/AssemblyInfo.cpp b/GVFS/GVFS.GvFltWrapper/AssemblyInfo.cpp index a17aa29d3f..364390669d 100644 --- a/GVFS/GVFS.GvFltWrapper/AssemblyInfo.cpp +++ b/GVFS/GVFS.GvFltWrapper/AssemblyInfo.cpp @@ -1,5 +1,5 @@ #include "stdafx.h" -#include "CommonAssemblyVersion.h" +#include "AssemblyVersion.h" using namespace System; using namespace System::Reflection; @@ -12,11 +12,11 @@ using namespace System::Security::Permissions; // set of attributes. Change these attribute values to modify the information // associated with an assembly. // -[assembly:AssemblyTitleAttribute(L"GVFSGvFltWrapper")]; +[assembly:AssemblyTitleAttribute(L"GvFlt")]; [assembly:AssemblyDescriptionAttribute(L"")]; [assembly:AssemblyConfigurationAttribute(L"")]; [assembly:AssemblyCompanyAttribute(L"")]; -[assembly:AssemblyProductAttribute(L"GVFSGvFltWrapper")]; +[assembly:AssemblyProductAttribute(L"GvFlt")]; [assembly:AssemblyCopyrightAttribute(L"Copyright (c) Microsoft 2017")]; [assembly:AssemblyTrademarkAttribute(L"")]; [assembly:AssemblyCultureAttribute(L"")]; diff --git a/GVFS/GVFS.GvFltWrapper/CallbackDelegates.h b/GVFS/GVFS.GvFltWrapper/CallbackDelegates.h new file mode 100644 index 0000000000..a56cdc6b56 --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/CallbackDelegates.h @@ -0,0 +1,287 @@ +#pragma once + +#include "DirectoryEnumerationResult.h" +#include "NtStatus.h" +#include "WriteBuffer.h" + +namespace GvFlt +{ + /// Directory is about to be enumerated + /// The Guid value associated with this directory path for a set of enumeration commands + /// The path (relative to the virtualization root) to be enumerated + /// Success if callback succeeded, appropriate error otherwise + public delegate NtStatus StartDirectoryEnumerationEvent( + System::Guid enumerationId, + System::String^ relativePath); + + /// Directory enumeration is complete + /// The Guid value associated with this directory path for a set of enumeration commands + /// Success if callback succeeded, appropriate error otherwise + public delegate NtStatus EndDirectoryEnumerationEvent(System::Guid enumerationId); + + /// Gets the next DirectoryEnumerationResult for the specified enumeration + /// The Guid value to associate with this directory path for a set of enumeration commands + /// + /// An optional string containing the name of a file (or multiple files, if wildcards are used) within the directory. + /// This parameter is optional and can be nullptr. + /// If filterFileName is not nullptr, only files whose names match the filterFileName string should be included in the directory scan. + /// If filterFileName is nullptr, all files should be included + /// + /// true if the scan is to start at the first entry in the directory. false if resuming the scan from a previous call. + /// Next DirectoryEnumerationResult to return in the enumeration result + /// + /// NtStatus indicating the result of the callback: + /// + /// Success - result set successfully + /// NoSuchFile - No file matches the specified filter. (Note: NoSuchFile should only be returned when filterFileName is set + /// (i.e.non - empty and not '*') and only for the first request for the specified filterFileName) + /// NoMoreFiles - No more files match the specified filter (or if no filter is set, there are no files) + /// BufferOverflow - File name of the next result does not fit in result (i.e. DirectoryEnumerationResult.TrySetFileName failed). + /// (Or appropriate error in case of failure) + /// + /// + /// - filterFileName must be persisted across calls to GetDirectoryEnumerationEvent, and only be reset when restartScan is true + /// - If BufferOverflow is returned, the enumeration should not be advanced (and the next call to GetDirectoryEnumerationEvent + /// should return the entry that was previously too large to fit in the result) + /// + public delegate NtStatus GetDirectoryEnumerationEvent( + System::Guid enumerationId, + System::String^ filterFileName, + bool restartScan, + DirectoryEnumerationResult^ result); + + /// Checks if a relative file path exists in the provider's backing layer + /// The path (relative to the virtualization root) of the file being queried + /// + /// If relativePath exists in the provider's backing layer, return Success, otherwise return ObjectNameNotFound. + /// If a failure occurs, return the appropriate error. + /// + public delegate NtStatus QueryFileNameEvent(System::String^ relativePath); + + /// Request for placeholder information + /// The path (relative to the virtualization root) of the file to return information for + /// The requested access to the file or folder. See CreateFile API on MSDN for the possible values + /// The requested sharing mode of the file or folder + /// The requested create disposition + /// + /// The requested options to be applied when creating or opening the file. + /// Please refer to the MSDN page for NtCreateFile for more details and possible values for + /// DesiredAccess, ShareMode, CreateDisposition and CreateOptions parameters - + /// https://msdn.microsoft.com/en-us/library/bb432380(v=vs.85).aspx + /// + /// The PID for the process that triggered this callback + /// The image file name for triggeringProcessId + /// Success if callback succeeded, appropriate error otherwise + /// + /// In this callback, a single call to WritePlaceholderInformation should be made to send all the information + /// for creating the placeholder for the filename requested. Returning from callback signals the GvFlt driver that all + /// information needed to create a placeholder was provided to GvFlt by a successful call to WritePlaceholderInformation. + /// + public delegate NtStatus GetPlaceholderInformationEvent( + System::String^ relativePath, + unsigned long desiredAccess, + unsigned long shareMode, + unsigned long createDisposition, + unsigned long createOptions, + unsigned long triggeringProcessId, + System::String^ triggeringProcessImageFileName); + + /// Request for the file stream contents for creating the file stream on disk + /// + /// The path (relative to the virtualization root) of the file to return file contents for. If a file + /// has been renamed or moved, relativePath will be its original path (prior to move\rename). + /// + /// Requested byte offset of the stream content. Always 0 in the current version. + /// Requested number of bytes of the stream content. Always equal to the full stream length in the current version. + /// + /// The Guid value to associate with this file stream for a set of WriteFile commands. + /// The provider mush pass in this Guid value when calling WriteFile in the callback for the same file stream. + /// + /// ContentId of the placeholder + /// EpochId of the placeholder + /// The PID for the process that triggered this callback + /// The image file name for triggeringProcessId + /// Success if callback succeeded, appropriate error otherwise + /// + /// In this callback, the provider will make a single or multiple calls to WriteFile, to send the main file stream content for the file name requested. + /// Returning from the callback signals the GvFlt driver that all file stream content has been provided to GvFlt by a successful call to WriteFile. + /// + public delegate NtStatus GetFileStreamEvent( + System::String^ relativePath, + long long byteOffset, + unsigned long length, + System::Guid streamGuid, + array^ contentId, + array^ epochId, + unsigned long triggeringProcessId, + System::String^ triggeringProcessImageFileName); + + /// A file or folder has been written to for the first time + /// The path (relative to the virtualization root) of the file or folder + /// Success if callback succeeded, appropriate error otherwise + /// + /// Returning from the callback signals the GvFlt driver that the provider has completed all necessary + /// bookkeeping and the write operation can proceed. + /// + /// Note - + /// Returning an error from the callback will cause GvFlt driver to fail to proceed and therefore fail the user request + /// that triggers this callback. + /// + /// This callback is triggered on below operations - + /// Note - the callback is sent on an attempt to issue the operation to the file system, + /// not after the operation has actually succeeded. + /// + /// 1) Open(CreateFile) + /// a) A callback with the path to the target file / directory will be sent if FILE_WRITE_DATA, FILE_APPEND_DATA or + /// FILE_WRITE_ATTRIBUTES is set in the access mask + /// + /// 2) Delete(DeleteFile, RemoveDirectory or using DELETE_ON_CLOSE in CreateFile) + /// a) A callback with the path to the parent directory will be sent + /// + /// 3) Rename(rename, movefile) + /// a) A callback with the path to the source's parent directory will be sent + /// b) A callback with the path to the destination's parent directory will be sent + /// + /// 4) New file/folder(CreateFile) + /// a) A callback with the path to the parent directory will be sent + /// + /// GvFlt guarantees that below properties for this callback is true at all the times - + /// 1) The placeholder file for the path file name passed in in the callback must have been created on disk + /// 2) Only one callback will be sent for one file / directory during the lifetime of the virtualization root + /// (from the virtualization root folder is created to it's deleted) + /// + public delegate NtStatus NotifyFirstWriteEvent(System::String^ relativePath); + + /// A handle to a file or folder has been created + /// The path (relative to the virtualization root) of the file or folder + /// true if relativePath is for a folder, false if relativePath is for a file + /// Desired access specified for handle + /// Share mode specified for handle + /// Create disposition specified for handle + /// Create options specified for handle + /// Final completion status of the handle create operation + /// + /// [Out] A bit mask that indicates which notifications the provider wants to watch for the target file. + /// Refer to the NotificationType enum for a list of notifications the provider can watch. + /// If this field is not set, no notification will be watched. + /// + /// + /// Refer to the MSDN page for NtCreateFile for more details and possible values for + /// desiredAccess, shareMode, createDisposition, createOptions and ioStatusBlock parameters - + /// https://msdn.microsoft.com/en-us/library/bb432380(v=vs.85).aspx + /// + public delegate void NotifyFileHandleCreatedEvent( + System::String^ relativePath, + bool isDirectory, + unsigned long desiredAccess, + unsigned long shareMode, + unsigned long createDisposition, + unsigned long createOptions, + IoStatusBlockValue ioStatusBlock, + unsigned long% notificationMask); + + /// An attempt to delete a watched file or directory is made + /// The path (relative to the virtualization root) of the file or folder + /// true if relativePath is for a folder, false if relativePath is for a file + /// Success if the delete should be allowed to proceed, an error code if the delete should be prevented + /// + /// This is + /// 1) The pre-operation callback for FileDispositionInformation or + /// FileDispositionInformationEx with DeleteFile set to TRUE. + /// OR + /// 2) Pre-operation for IRP_MJ_CLEANUP if the handle was opened with DELETE_ON_CLOSE. + /// + /// GvFlt won't send this notification if someone tries to undo the delete by setting + /// DeleteFile to FALSE for FileDispositionInformation. + /// + public delegate NtStatus NotifyPreDeleteEvent( + System::String^ relativePath, + bool isDirectory); + + /// + /// An attempt to rename a watched file or directory, or move a file or directory + /// into the virtualization root is made + /// + /// The path (relative to the virtualization root) of the file or folder + /// Destination path (relative to the virtualization root) + /// Success if the rename should be allowed to proceed, an error code if the rename should be prevented + /// + /// This is the pre-operation callback for IRP_MJ_SET_INFORMATION with FileRenameInformation + /// or FileRenameInformationEx file information class. + /// + /// The provider won't receive the rename notification if GvFlt decided to fail the request. + /// eg. an attempt to rename a placeholder folder, or the path name to be renamed exists + /// in the backing layer. + /// + /// If moving a file from outside to inside the instance, relativePath will be "" + /// If moving a file from inside to outside the instance, destinationPath will be "" + /// If moving a file between 2 instances, each instance will receive one rename + /// notification if it's being watched, one with relativePath being "" and the other with + /// destinationPath being "". + /// + /// The provider won't receive any rename notification for renaming a stream. + /// + public delegate NtStatus NotifyPreRenameEvent( + System::String^ relativePath, + System::String^ destinationPath); + + /// An attempt to create a hardlink for a watched file is made. + /// The path (relative to the virtualization root) of the file + /// Destination file name + /// Success if the hardlink operation should be allowed to proceed, an error code if the operation should be prevented + /// This is the pre-operation callback for IRP_MJ_SET_INFORMATION with FileLinkInformation file information class + public delegate NtStatus NotifyPreSetHardlinkEvent( + System::String^ relativePath, + System::String^ destinationPath); + + /// A watched file or directory was renamed, or a file or directory was moved into the virtualization root + /// The path (relative to the virtualization root) of the file or folder + /// Destination path (relative to the virtualization root) + /// true if relativePath is for a folder, false if relativePath is for a file + /// + /// [InOut] A bit mask that indicates which notifications the provide wants to watch for the renamed file. + /// If this field is not set, the notification mask of the source file will be used. + /// + /// + /// Paths will be empty strings when location is outside of the virtualization root. + /// This is the post-operation callback for IRP_MJ_SET_INFORMATION with FileRenameInformation + /// or FileRenameInformationEx file information class. If the rename opeartion failed, eg. the destination file already exists, + /// the provider won't receive this notification. + /// + public delegate void NotifyFileRenamedEvent( + System::String^ relativePath, + System::String^ destinationPath, + bool isDirectory, + unsigned long% notificationMask); + + /// A hardlink within the virtualization root was created + /// The path (relative to the virtualization root) of the file or folder + /// Destination file name + /// This is the post-operation callback for IRP_MJ_SET_INFORMATION with FileLinkInformation file information class + public delegate void NotifyHardlinkCreatedEvent( + System::String^ relativePath, + System::String^ destinationPath); + + /// A handle to a watched file or directory was closed + /// The path (relative to the virtualization root) of the file or folder + /// true if relativePath is for a folder, false if relativePath is for a file + /// If true, a handle which was used to modify the file's main stream data was closed. + /// If true, the file has been deleted from the file system + /// + /// fileModified is set to true if: + /// 1) A cached write was made using the handle. + /// Or 2) A non-cached write was made using the handle. + /// Or 3) A writable section was created using the handle. + /// + /// fileDeleted requires that the OS support the 'reliable delete information' feature, otherwise it will always be false. + /// To check if the target volume supports this feature, the provider can retrieve the volume info and query if the flag below is set. + /// See also https://msdn.microsoft.com/en-us/library/windows/desktop/aa364993(v=vs.85).aspx + /// + /// #define FILE_RETURNS_CLEANUP_RESULT_INFO 0x00000200 // winnt + /// + public delegate void NotifyFileHandleClosedEvent( + System::String^ relativePath, + bool isDirectory, + bool fileModified, + bool fileDeleted); +} \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationFileNamesResult.h b/GVFS/GVFS.GvFltWrapper/DirectoryEnumerationFileNamesResult.h similarity index 72% rename from GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationFileNamesResult.h rename to GVFS/GVFS.GvFltWrapper/DirectoryEnumerationFileNamesResult.h index baeaeef5b1..8b6a222467 100644 --- a/GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationFileNamesResult.h +++ b/GVFS/GVFS.GvFltWrapper/DirectoryEnumerationFileNamesResult.h @@ -1,14 +1,14 @@ #pragma once -#include "GvDirectoryEnumerationResult.h" +#include "DirectoryEnumerationResult.h" #include "NativeEnumerationResultUtils.h" -namespace GVFSGvFltWrapper +namespace GvFlt { - public ref class GvDirectoryEnumerationFileNamesResult : public GvDirectoryEnumerationResult + public ref class DirectoryEnumerationFileNamesResult : public DirectoryEnumerationResult { public: - GvDirectoryEnumerationFileNamesResult(PFILE_NAMES_INFORMATION enumerationData, unsigned long maxEnumerationDataLength); + DirectoryEnumerationFileNamesResult(PFILE_NAMES_INFORMATION enumerationData, unsigned long maxEnumerationDataLength); property System::DateTime CreationTime { @@ -48,14 +48,14 @@ namespace GVFSGvFltWrapper }; - inline GvDirectoryEnumerationFileNamesResult::GvDirectoryEnumerationFileNamesResult(PFILE_NAMES_INFORMATION enumerationData, unsigned long maxEnumerationDataLength) + inline DirectoryEnumerationFileNamesResult::DirectoryEnumerationFileNamesResult(PFILE_NAMES_INFORMATION enumerationData, unsigned long maxEnumerationDataLength) : enumerationData(enumerationData) , maxEnumerationDataLength(maxEnumerationDataLength) { this->bytesWritten = FIELD_OFFSET(FILE_NAMES_INFORMATION, FileName); } - inline bool GvDirectoryEnumerationFileNamesResult::TrySetFileName(System::String^ value) + inline bool DirectoryEnumerationFileNamesResult::TrySetFileName(System::String^ value) { bool nameTruncated = false; this->bytesWritten = PopulateNameInEnumerationData(this->enumerationData, this->maxEnumerationDataLength, value, nameTruncated); diff --git a/GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationResult.h b/GVFS/GVFS.GvFltWrapper/DirectoryEnumerationResult.h similarity index 78% rename from GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationResult.h rename to GVFS/GVFS.GvFltWrapper/DirectoryEnumerationResult.h index 5482d97b44..7f3e69b5a0 100644 --- a/GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationResult.h +++ b/GVFS/GVFS.GvFltWrapper/DirectoryEnumerationResult.h @@ -1,11 +1,11 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { - public ref class GvDirectoryEnumerationResult abstract + public ref class DirectoryEnumerationResult abstract { public: - GvDirectoryEnumerationResult(); + DirectoryEnumerationResult(); property System::DateTime CreationTime { @@ -51,12 +51,12 @@ namespace GVFSGvFltWrapper }; - inline GvDirectoryEnumerationResult::GvDirectoryEnumerationResult() + inline DirectoryEnumerationResult::DirectoryEnumerationResult() : bytesWritten(0) { } - inline unsigned long GvDirectoryEnumerationResult::BytesWritten::get(void) + inline unsigned long DirectoryEnumerationResult::BytesWritten::get(void) { return this->bytesWritten; } diff --git a/GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationResultImpl.h b/GVFS/GVFS.GvFltWrapper/DirectoryEnumerationResultImpl.h similarity index 63% rename from GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationResultImpl.h rename to GVFS/GVFS.GvFltWrapper/DirectoryEnumerationResultImpl.h index de8f926f02..8a9e5b80df 100644 --- a/GVFS/GVFS.GvFltWrapper/GvDirectoryEnumerationResultImpl.h +++ b/GVFS/GVFS.GvFltWrapper/DirectoryEnumerationResultImpl.h @@ -1,14 +1,14 @@ #pragma once -#include "GvDirectoryEnumerationResult.h" +#include "DirectoryEnumerationResult.h" -namespace GVFSGvFltWrapper +namespace GvFlt { template - public ref class GvDirectoryEnumerationResultImpl : public GvDirectoryEnumerationResult + public ref class DirectoryEnumerationResultImpl : public DirectoryEnumerationResult { public: - GvDirectoryEnumerationResultImpl(NativeEnumerationDataStruct* enumerationData, unsigned long maxEnumerationDataLength); + DirectoryEnumerationResultImpl(NativeEnumerationDataStruct* enumerationData, unsigned long maxEnumerationDataLength); property System::DateTime CreationTime { @@ -49,7 +49,7 @@ namespace GVFSGvFltWrapper template - inline GvDirectoryEnumerationResultImpl::GvDirectoryEnumerationResultImpl(NativeEnumerationDataStruct* enumerationData, unsigned long maxEnumerationDataLength) + inline DirectoryEnumerationResultImpl::DirectoryEnumerationResultImpl(NativeEnumerationDataStruct* enumerationData, unsigned long maxEnumerationDataLength) : enumerationData(enumerationData) , maxEnumerationDataLength(maxEnumerationDataLength) { @@ -60,43 +60,43 @@ namespace GVFSGvFltWrapper } template - inline void GvDirectoryEnumerationResultImpl::CreationTime::set(System::DateTime value) + inline void DirectoryEnumerationResultImpl::CreationTime::set(System::DateTime value) { this->enumerationData->CreationTime.QuadPart = value.ToFileTime(); } template - inline void GvDirectoryEnumerationResultImpl::LastAccessTime::set(System::DateTime value) + inline void DirectoryEnumerationResultImpl::LastAccessTime::set(System::DateTime value) { this->enumerationData->LastAccessTime.QuadPart = value.ToFileTime(); } template - inline void GvDirectoryEnumerationResultImpl::LastWriteTime::set(System::DateTime value) + inline void DirectoryEnumerationResultImpl::LastWriteTime::set(System::DateTime value) { this->enumerationData->LastWriteTime.QuadPart = value.ToFileTime(); } template - inline void GvDirectoryEnumerationResultImpl::ChangeTime::set(System::DateTime value) + inline void DirectoryEnumerationResultImpl::ChangeTime::set(System::DateTime value) { this->enumerationData->ChangeTime.QuadPart = value.ToFileTime(); } template - inline void GvDirectoryEnumerationResultImpl::EndOfFile::set(long long value) + inline void DirectoryEnumerationResultImpl::EndOfFile::set(long long value) { this->enumerationData->EndOfFile.QuadPart = value; } template - inline void GvDirectoryEnumerationResultImpl::FileAttributes::set(unsigned int value) + inline void DirectoryEnumerationResultImpl::FileAttributes::set(unsigned int value) { this->enumerationData->FileAttributes = value; } template - inline bool GvDirectoryEnumerationResultImpl::TrySetFileName(System::String^ value) + inline bool DirectoryEnumerationResultImpl::TrySetFileName(System::String^ value) { bool nameTruncated = false; this->bytesWritten = PopulateNameInEnumerationData(this->enumerationData, this->maxEnumerationDataLength, value, nameTruncated); diff --git a/GVFS/GVFS.GvFltWrapper/GvFlt.props b/GVFS/GVFS.GvFltWrapper/GvFlt.props index 2d16185ecc..f8bd753973 100644 --- a/GVFS/GVFS.GvFltWrapper/GvFlt.props +++ b/GVFS/GVFS.GvFltWrapper/GvFlt.props @@ -2,7 +2,7 @@ - Microsoft.GVFS.GvFlt.0.17627.2-preview + Microsoft.GVFS.GvFlt.0.17817.1-preview diff --git a/GVFS/GVFS.GvFltWrapper/GVFS.GvFltWrapper.vcxproj b/GVFS/GVFS.GvFltWrapper/GvFlt.vcxproj similarity index 78% rename from GVFS/GVFS.GvFltWrapper/GVFS.GvFltWrapper.vcxproj rename to GVFS/GVFS.GvFltWrapper/GvFlt.vcxproj index 9924935daf..10feadabd2 100644 --- a/GVFS/GVFS.GvFltWrapper/GVFS.GvFltWrapper.vcxproj +++ b/GVFS/GVFS.GvFltWrapper/GvFlt.vcxproj @@ -14,7 +14,7 @@ {FB0831AE-9997-401B-B31F-3A065FDBEB20} v4.5.2 ManagedCProj - GVFSGvFltWrapper + GvFlt 10.0.10240.0 @@ -67,7 +67,8 @@ _DEBUG;%(PreprocessorDefinitions) Use true - C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;$(SolutionDir)\..\packages\$(GvFltPackage)\header;$(SolutionDir)\..\BuildOutput + C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;$(SolutionDir)\..\packages\$(GvFltPackage)\header;$(SolutionDir)\..\BuildOutput\$(ProjectName) + true gvlib.lib;fltlib.lib;Ole32.lib;Advapi32.lib @@ -80,10 +81,11 @@ $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log - $(SolutionDir)\Scripts\CreateCommonCliAssemblyVersion.bat $(GVFSVersion) $(SolutionDir)\.. + $(ProjectDir)\Scripts\CreateVersionHeader.bat $(GVFSVersion) $(SolutionDir)\.. && $(ProjectDir)\Scripts\CreateCliAssemblyVersion.bat $(GVFSVersion) $(SolutionDir)\.. + - $(SolutionDir)\..\BuildOutput + $(SolutionDir)\..\BuildOutput\$(ProjectName) @@ -92,7 +94,8 @@ NDEBUG;%(PreprocessorDefinitions) Use true - C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;$(SolutionDir)\..\packages\$(GvFltPackage)\header;$(SolutionDir)\..\BuildOutput + C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;$(SolutionDir)\..\packages\$(GvFltPackage)\header;$(SolutionDir)\..\BuildOutput\$(ProjectName) + true gvlib.lib;fltlib.lib;Ole32.lib;Advapi32.lib @@ -103,42 +106,48 @@ $(SolutionDir)..\BuildOutput\$(ProjectName)\intermediate\$(Platform)\$(Configuration)\$(MSBuildProjectName).log - $(SolutionDir)\Scripts\CreateCommonCliAssemblyVersion.bat $(GVFSVersion) $(SolutionDir)\.. + $(ProjectDir)\Scripts\CreateVersionHeader.bat $(GVFSVersion) $(SolutionDir)\.. && $(ProjectDir)\Scripts\CreateCliAssemblyVersion.bat $(GVFSVersion) $(SolutionDir)\.. + - $(SolutionDir)\..\BuildOutput + $(SolutionDir)\..\BuildOutput\$(ProjectName) - + + False + ..\..\..\packages\Microsoft.Diagnostics.Tracing.EventSource.Redist.1.1.28\lib\net40\Microsoft.Diagnostics.Tracing.EventSource.dll + True + - - - - + + + + - - - - - + + + - + + + + - - + + Create Create @@ -149,11 +158,8 @@ Designer - - - - {374bf1e5-0b2d-4d4a-bd5e-4212299def09} - + + diff --git a/GVFS/GVFS.GvFltWrapper/GVFS.GvFltWrapper.vcxproj.filters b/GVFS/GVFS.GvFltWrapper/GvFlt.vcxproj.filters similarity index 69% rename from GVFS/GVFS.GvFltWrapper/GVFS.GvFltWrapper.vcxproj.filters rename to GVFS/GVFS.GvFltWrapper/GvFlt.vcxproj.filters index e7b9a2d157..3b3adb0b04 100644 --- a/GVFS/GVFS.GvFltWrapper/GVFS.GvFltWrapper.vcxproj.filters +++ b/GVFS/GVFS.GvFltWrapper/GvFlt.vcxproj.filters @@ -12,57 +12,63 @@ {c42f0003-79e9-4a34-9c03-c74c9242f1d8} + + {5faad13c-1a12-4726-955f-2ffb51b701fa} + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + Header Files - + + Header Files + + Header Files @@ -73,18 +79,24 @@ Source Files - + Source Files - + Source Files - + Source Files + + Scripts + + + Scripts + diff --git a/GVFS/GVFS.GvFltWrapper/GvFltCallbackDelegates.h b/GVFS/GVFS.GvFltWrapper/GvFltCallbackDelegates.h deleted file mode 100644 index 937c71dc16..0000000000 --- a/GVFS/GVFS.GvFltWrapper/GvFltCallbackDelegates.h +++ /dev/null @@ -1,66 +0,0 @@ -#pragma once - -#include "GvDirectoryEnumerationResult.h" -#include "GVFltWriteBuffer.h" - -namespace GVFSGvFltWrapper -{ - public delegate StatusCode GvStartDirectoryEnumerationEvent(System::Guid enumerationId, System::String^ relativePath); - - public delegate StatusCode GvEndDirectoryEnumerationEvent(System::Guid enumerationId); - - public delegate StatusCode GvGetDirectoryEnumerationEvent( - System::Guid enumerationId, - System::String^ filterFileName, - bool restartScan, - GvDirectoryEnumerationResult^ result); - - public delegate StatusCode GvQueryFileNameEvent(System::String^ relativePath); - - public delegate StatusCode GvGetPlaceHolderInformationEvent( - System::String^ relativePath, - unsigned long desiredAccess, - unsigned long shareMode, - unsigned long createDisposition, - unsigned long createOptions, - unsigned long triggeringProcessId, - System::String^ triggeringProcessImageFileName); - - public delegate StatusCode GvGetFileStreamEvent( - System::String^ relativePath, - long long byteOffset, - unsigned long length, - System::Guid streamGuid, - System::String^ contentId, - unsigned long triggeringProcessId, - System::String^ triggeringProcessImageFileName, - GVFltWriteBuffer^ targetBuffer); - - public delegate StatusCode GvNotifyFirstWriteEvent(System::String^ relativePath); - - public delegate void GvNotifyCreateEvent( - System::String^ relativePath, - bool isDirectory, - unsigned long desiredAccess, - unsigned long shareMode, - unsigned long createDisposition, - unsigned long createOptions, - IoStatusBlockValue ioStatusBlock, - unsigned long% notificationMask); - - public delegate StatusCode GvNotifyPreDeleteEvent(System::String^ relativePath, bool isDirectory); - - public delegate StatusCode GvNotifyPreRenameEvent(System::String^ relativePath, System::String^ destinationPath); - - public delegate StatusCode GvNotifyPreSetHardlinkEvent(System::String^ relativePath, System::String^ destinationPath); - - public delegate void GvNotifyFileRenamedEvent(System::String^ relativePath, System::String^ destinationPath, bool isDirectory, unsigned long% notificationMask); - - public delegate void GvNotifyHardlinkCreatedEvent(System::String^ relativePath, System::String^ destinationPath); - - public delegate void GvNotifyFileHandleClosedEvent( - System::String^ relativePath, - bool isDirectory, - bool fileModified, - bool fileDeleted); -} \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/GvFltException.cpp b/GVFS/GVFS.GvFltWrapper/GvFltException.cpp index 449af2b5f0..7ee747e145 100644 --- a/GVFS/GVFS.GvFltWrapper/GvFltException.cpp +++ b/GVFS/GVFS.GvFltWrapper/GvFltException.cpp @@ -2,19 +2,19 @@ #include "GvFltException.h" using namespace System; -using namespace GVFSGvFltWrapper; +using namespace GvFlt; GvFltException::GvFltException(String^ errorMessage) - : GvFltException(errorMessage, StatusCode::StatusInternalError) + : GvFltException(errorMessage, NtStatus::InternalError) { } -GvFltException::GvFltException(StatusCode errorCode) +GvFltException::GvFltException(NtStatus errorCode) : GvFltException("GvFltException exception, error: " + errorCode.ToString(), errorCode) { } -GvFltException::GvFltException(String^ errorMessage, StatusCode errorCode) +GvFltException::GvFltException(String^ errorMessage, NtStatus errorCode) : Exception(errorMessage) , errorCode(errorCode) { @@ -25,7 +25,7 @@ String^ GvFltException::ToString() return String::Format("GvFltException ErrorCode: {0}, {1}", + this->errorCode, this->Exception::ToString()); } -StatusCode GvFltException::ErrorCode::get(void) +NtStatus GvFltException::ErrorCode::get(void) { return this->errorCode; }; \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/GvFltException.h b/GVFS/GVFS.GvFltWrapper/GvFltException.h index a80c7b91b1..e0f9dc26f3 100644 --- a/GVFS/GVFS.GvFltWrapper/GvFltException.h +++ b/GVFS/GVFS.GvFltWrapper/GvFltException.h @@ -1,24 +1,25 @@ #pragma once -#include "StatusCode.h" -namespace GVFSGvFltWrapper +#include "NtStatus.h" + +namespace GvFlt { [System::Serializable()] public ref class GvFltException : System::Exception { public: GvFltException(System::String^ errorMessage); - GvFltException(StatusCode errorCode); - GvFltException(System::String^ errorMessage, StatusCode errorCode); + GvFltException(NtStatus errorCode); + GvFltException(System::String^ errorMessage, NtStatus errorCode); virtual System::String^ ToString() override; - virtual property StatusCode ErrorCode + virtual property NtStatus ErrorCode { - StatusCode get(void); + NtStatus get(void); }; private: - StatusCode errorCode; + NtStatus errorCode; }; } \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/GvFltWrapper.h b/GVFS/GVFS.GvFltWrapper/GvFltWrapper.h deleted file mode 100644 index 5904280ef9..0000000000 --- a/GVFS/GVFS.GvFltWrapper/GvFltWrapper.h +++ /dev/null @@ -1,211 +0,0 @@ -#pragma once - -#include "GvFltCallbackDelegates.h" -#include "GvUpdateFailureCause.h" -#include "HResult.h" - -namespace GVFSGvFltWrapper -{ - public ref class GvFltWrapper - { - public: - GvFltWrapper(); - - property GvStartDirectoryEnumerationEvent^ OnStartDirectoryEnumeration - { - GvStartDirectoryEnumerationEvent^ get(void); - void set(GvStartDirectoryEnumerationEvent^ eventCB); - }; - - property GvEndDirectoryEnumerationEvent^ OnEndDirectoryEnumeration - { - GvEndDirectoryEnumerationEvent^ get(void); - void set(GvEndDirectoryEnumerationEvent^ eventCB); - }; - - property GvGetDirectoryEnumerationEvent^ OnGetDirectoryEnumeration - { - GvGetDirectoryEnumerationEvent^ get(void); - void set(GvGetDirectoryEnumerationEvent^ eventCB); - }; - - property GvQueryFileNameEvent^ OnQueryFileName - { - GvQueryFileNameEvent^ get(void); - void set(GvQueryFileNameEvent^ eventCB); - } - - property GvGetPlaceHolderInformationEvent^ OnGetPlaceHolderInformation - { - GvGetPlaceHolderInformationEvent^ get(void); - void set(GvGetPlaceHolderInformationEvent^ eventCB); - }; - - property GvGetFileStreamEvent^ OnGetFileStream - { - GvGetFileStreamEvent^ get(void); - void set(GvGetFileStreamEvent^ eventCB); - }; - - property GvNotifyFirstWriteEvent^ OnNotifyFirstWrite - { - GvNotifyFirstWriteEvent^ get(void); - void set(GvNotifyFirstWriteEvent^ eventCB); - }; - - property GvNotifyCreateEvent^ OnNotifyCreate - { - GvNotifyCreateEvent^ get(void); - void set(GvNotifyCreateEvent^ eventCB); - }; - - property GvNotifyPreDeleteEvent^ OnNotifyPreDelete - { - GvNotifyPreDeleteEvent^ get(void); - void set(GvNotifyPreDeleteEvent^ eventCB); - } - - property GvNotifyPreRenameEvent^ OnNotifyPreRename - { - GvNotifyPreRenameEvent^ get(void); - void set(GvNotifyPreRenameEvent^ eventCB); - } - - property GvNotifyPreSetHardlinkEvent^ OnNotifyPreSetHardlink - { - GvNotifyPreSetHardlinkEvent^ get(void); - void set(GvNotifyPreSetHardlinkEvent^ eventCB); - } - - property GvNotifyFileRenamedEvent^ OnNotifyFileRenamed - { - GvNotifyFileRenamedEvent^ get(void); - void set(GvNotifyFileRenamedEvent^ eventCB); - } - - property GvNotifyHardlinkCreatedEvent^ OnNotifyHardlinkCreated - { - GvNotifyHardlinkCreatedEvent^ get(void); - void set(GvNotifyHardlinkCreatedEvent^ eventCB); - } - - property GvNotifyFileHandleClosedEvent^ OnNotifyFileHandleClosed - { - GvNotifyFileHandleClosedEvent^ get(void); - void set(GvNotifyFileHandleClosedEvent^ eventCB); - } - - property GVFS::Common::Tracing::ITracer^ Tracer - { - GVFS::Common::Tracing::ITracer^ get(void); - }; - - HResult GvStartVirtualizationInstance( - GVFS::Common::Tracing::ITracer^ tracerImpl, - System::String^ virtualizationRootPath, - unsigned long poolThreadCount, - unsigned long concurrentThreadCount); - - HResult GvStopVirtualizationInstance(); - - HResult GvDetachDriver(); - - StatusCode GvWriteFile( - System::Guid streamGuid, - GVFltWriteBuffer^ targetBuffer, - unsigned long long byteOffset, - unsigned long length); - - StatusCode GvDeleteFile(System::String^ relativePath, GvUpdateType updateFlags, GvUpdateFailureCause% failureReason); - - StatusCode GvWritePlaceholderInformation( - System::String^ targetRelPathName, - System::DateTime creationTime, - System::DateTime lastAccessTime, - System::DateTime lastWriteTime, - System::DateTime changeTime, - unsigned long fileAttributes, - long long endOfFile, - bool directory, - System::String^ contentId, - System::String^ epochId); - - StatusCode GvCreatePlaceholderAsHardlink( - System::String^ destinationFileName, - System::String^ hardLinkTarget); - - StatusCode GvUpdatePlaceholderIfNeeded( - System::String^ relativePath, - System::DateTime creationTime, - System::DateTime lastAccessTime, - System::DateTime lastWriteTime, - System::DateTime changeTime, - unsigned long fileAttributes, - long long endOfFile, - System::String^ contentId, - System::String^ epochId, - GvUpdateType updateFlags, - GvUpdateFailureCause% failureReason); - - enum class OnDiskStatus : long - { - NotOnDisk = 0, - Partial = 1, - Full = 2, - OnDiskCannotOpen = 3 - }; - - // FileExists - // - // Returns: - // OnDiskStatus indicating if the file is not on disk, a partial file, or a full file. - // - // Throws: - // GvFltException - // - // Notes: - // This function cannot be used to determine if a folder is partial or full, and cannot be - // used to determine if a path is a file or a folder. - OnDiskStatus GetFileOnDiskStatus(System::String^ relativePath); - - // ReadFullFileContents - // - // Returns: - // Contents of the specified full file. BOM, if present, is not removed. - // - // Throws: - // - GvFltException - System::String^ ReadFullFileContents(System::String^ relativePath); - - ULONG GetWriteBufferSize(); - ULONG GetAlignmentRequirement(); - - static HResult GvConvertDirectoryToVirtualizationRoot(System::Guid virtualizationInstanceGuid, System::String^ rootPathName); - - private: - void ConfirmNotStarted(); - void CalculateWriteBufferSizeAndAlignment(); - - GvStartDirectoryEnumerationEvent^ gvStartDirectoryEnumerationEvent; - GvEndDirectoryEnumerationEvent^ gvEndDirectoryEnumerationEvent; - GvGetDirectoryEnumerationEvent^ gvGetDirectoryEnumerationEvent; - GvQueryFileNameEvent^ gvQueryFileNameEvent; - GvGetPlaceHolderInformationEvent^ gvGetPlaceHolderInformationEvent; - GvGetFileStreamEvent^ gvGetFileStreamEvent; - GvNotifyFirstWriteEvent^ gvNotifyFirstWriteEvent; - GvNotifyCreateEvent^ gvNotifyCreateEvent; - GvNotifyPreDeleteEvent^ gvNotifyPreDeleteEvent; - GvNotifyPreRenameEvent^ gvNotifyPreRenameEvent; - GvNotifyPreSetHardlinkEvent^ gvNotifyPreSetHardlinkEvent; - GvNotifyFileRenamedEvent^ gvNotifyFileRenamedEvent; - GvNotifyHardlinkCreatedEvent^ gvNotifyHardlinkCreatedEvent; - GvNotifyFileHandleClosedEvent^ gvNotifyFileHandleClosedEvent; - - ULONG writeBufferSize; - ULONG alignmentRequirement; - - GV_VIRTUALIZATIONINSTANCE_HANDLE virtualizationInstanceHandle; - System::String^ virtualRootPath; - GVFS::Common::Tracing::ITracer^ tracer; - }; -} diff --git a/GVFS/GVFS.GvFltWrapper/GvNotificationType.h b/GVFS/GVFS.GvFltWrapper/GvNotificationType.h deleted file mode 100644 index 6e0721fba7..0000000000 --- a/GVFS/GVFS.GvFltWrapper/GvNotificationType.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -namespace GVFSGvFltWrapper -{ - [System::FlagsAttribute] - public enum class GvNotificationType : unsigned long - { - NotificationNone = GV_NOTIFICATION_NONE, - NotificationPostCreate = GV_NOTIFICATION_POST_CREATE, - NotificationPreDelete = GV_NOTIFICATION_PRE_DELETE, - NotificationPreRename = GV_NOTIFICATION_PRE_RENAME, - NotificationPreSetHardlink = GV_NOTIFICATION_PRE_SET_HARDLINK, - NotificationFileRenamed = GV_NOTIFICATION_FILE_RENAMED, - NotificationHardlinkCreated = GV_NOTIFICATION_HARDLINK_CREATED, - NotificationFileHandleClosed = GV_NOTIFICATION_FILE_HANDLE_CLOSED, - }; -} diff --git a/GVFS/GVFS.GvFltWrapper/GvUpdateType.h b/GVFS/GVFS.GvFltWrapper/GvUpdateType.h deleted file mode 100644 index 282f36af17..0000000000 --- a/GVFS/GVFS.GvFltWrapper/GvUpdateType.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -namespace GVFSGvFltWrapper -{ - [System::FlagsAttribute] - public enum class GvUpdateType : unsigned long - { - UpdateAllowDirtyMetadata = GV_UPDATE_ALLOW_DIRTY_METADATA, - UpdateAllowDirtyData = GV_UPDATE_ALLOW_DIRTY_DATA, - UpdateAllowTombstone = GV_UPDATE_ALLOW_TOMBSTONE, - UpdateAllowReadOnly = GV_UPDATE_ALLOW_READ_ONLY - }; -} diff --git a/GVFS/GVFS.GvFltWrapper/HResult.h b/GVFS/GVFS.GvFltWrapper/HResult.h index 24bbfde680..45655ab23a 100644 --- a/GVFS/GVFS.GvFltWrapper/HResult.h +++ b/GVFS/GVFS.GvFltWrapper/HResult.h @@ -1,22 +1,24 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { public enum class HResult : long { // Subset of HRESULT values. Add more values as needed. - Ok = S_OK, // Operation successful - Abort = E_ABORT, // Operation aborted - AccessDenied = E_ACCESSDENIED, // General access denied error - Fail = E_FAIL, // Unspecified failure - Handle = E_HANDLE, // Handle that is not valid - InvalidArg = E_INVALIDARG, // One or more arguments are not valid - NoInterface = E_NOINTERFACE, // No such interface supported - NotImpl = E_NOTIMPL, // Not implemented - OutOfMemory = E_OUTOFMEMORY, // Failed to allocate necessary memory - Pointer = E_POINTER, // Pointer that is not valid - Unexpected = E_UNEXPECTED, // Unexpected failure + Ok = S_OK, // Operation successful + Abort = E_ABORT, // Operation aborted + AccessDenied = E_ACCESSDENIED, // General access denied error + Fail = E_FAIL, // Unspecified failure + Handle = E_HANDLE, // Handle that is not valid + InvalidArg = E_INVALIDARG, // One or more arguments are not valid + NoInterface = E_NOINTERFACE, // No such interface supported + NotImpl = E_NOTIMPL, // Not implemented + OutOfMemory = E_OUTOFMEMORY, // Failed to allocate necessary memory + Pointer = E_POINTER, // Pointer that is not valid + Unexpected = E_UNEXPECTED, // Unexpected failure + PrivilegeNotHeld = ERROR_PRIVILEGE_NOT_HELD, // A required privilege is not held by the client. ReparsePointEncountered = __HRESULT_FROM_WIN32(ERROR_REPARSE_POINT_ENCOUNTERED) // The object manager encountered a reparse point while retrieving an object. + }; } \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/ITracer.h b/GVFS/GVFS.GvFltWrapper/ITracer.h new file mode 100644 index 0000000000..325095b979 --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/ITracer.h @@ -0,0 +1,30 @@ +#pragma once + +namespace GvFlt +{ + public interface class ITracer + { + // Trace an event + // + // Parameters: + // level: EventLevel of the event + // eventName: Name of the event + // metadata: Key\value pairs of event data to be recorded + void TraceEvent( + Microsoft::Diagnostics::Tracing::EventLevel level, + System::String^ eventName, + System::Collections::Generic::Dictionary^ metadata); + + // Trace an error + // + // Parameters: + // message: Error message to record + void TraceError(System::String^ message); + + // Trace an error + // + // Parameters: + // metadata: Key\value pairs of error data to be recorded + void TraceError(System::Collections::Generic::Dictionary^ metadata); + }; +} \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/IoStatusBlockValue.h b/GVFS/GVFS.GvFltWrapper/IoStatusBlockValue.h index 6596078379..7782a162ef 100644 --- a/GVFS/GVFS.GvFltWrapper/IoStatusBlockValue.h +++ b/GVFS/GVFS.GvFltWrapper/IoStatusBlockValue.h @@ -1,6 +1,6 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { public enum class IoStatusBlockValue : unsigned int { diff --git a/GVFS/GVFS.GvFltWrapper/NativeEnumerationResultUtils.h b/GVFS/GVFS.GvFltWrapper/NativeEnumerationResultUtils.h index 929106875e..955b055cba 100644 --- a/GVFS/GVFS.GvFltWrapper/NativeEnumerationResultUtils.h +++ b/GVFS/GVFS.GvFltWrapper/NativeEnumerationResultUtils.h @@ -1,6 +1,6 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { // PopulateNameInEnumerationData // diff --git a/GVFS/GVFS.GvFltWrapper/NotificationType.h b/GVFS/GVFS.GvFltWrapper/NotificationType.h new file mode 100644 index 0000000000..a731f4b26d --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/NotificationType.h @@ -0,0 +1,17 @@ +#pragma once + +namespace GvFlt +{ + [System::FlagsAttribute] + public enum class NotificationType : unsigned long + { + None = GV_NOTIFICATION_NONE, + PostCreate = GV_NOTIFICATION_POST_CREATE, + PreDelete = GV_NOTIFICATION_PRE_DELETE, + PreRename = GV_NOTIFICATION_PRE_RENAME, + PreSetHardlink = GV_NOTIFICATION_PRE_SET_HARDLINK, + FileRenamed = GV_NOTIFICATION_FILE_RENAMED, + HardlinkCreated = GV_NOTIFICATION_HARDLINK_CREATED, + FileHandleClosed = GV_NOTIFICATION_FILE_HANDLE_CLOSED, + }; +} diff --git a/GVFS/GVFS.GvFltWrapper/NtStatus.h b/GVFS/GVFS.GvFltWrapper/NtStatus.h new file mode 100644 index 0000000000..aea73f9fe3 --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/NtStatus.h @@ -0,0 +1,39 @@ +#pragma once + +namespace GvFlt +{ + public enum class NtStatus : long + { + // Subset of NTSTATUS values. Add more values as needed. + Succcess = STATUS_SUCCESS, + Timeout = STATUS_TIMEOUT, + FileNotAvailable = STATUS_FILE_NOT_AVAILABLE, + Unsuccessful = STATUS_UNSUCCESSFUL, + NotImplemented = STATUS_NOT_IMPLEMENTED, + InvalidHandle = STATUS_INVALID_HANDLE, + InvalidParameter = STATUS_INVALID_PARAMETER, + ObjectNameNotFound = STATUS_OBJECT_NAME_NOT_FOUND, + ObjectPathNotFound = STATUS_OBJECT_PATH_NOT_FOUND, + InvalidDeviceRequest = STATUS_INVALID_DEVICE_REQUEST, + EndOfFile = STATUS_END_OF_FILE, + BufferOverflow = STATUS_BUFFER_OVERFLOW, + InternalError = STATUS_INTERNAL_ERROR, + NoMemory = STATUS_NO_MEMORY, + NoMoreFiles = STATUS_NO_MORE_FILES, + NoSuchFile = STATUS_NO_SUCH_FILE, + RequestAborted = STATUS_REQUEST_ABORTED, + AccessDenied = STATUS_ACCESS_DENIED, + NoInterface = STATUS_NOINTERFACE, + DeviceNotReady = STATUS_DEVICE_NOT_READY, + FileClosed = STATUS_FILE_CLOSED, + ObjectNameInvalid = STATUS_OBJECT_NAME_INVALID, + DirectoryNotEmpty = STATUS_DIRECTORY_NOT_EMPTY, + CannotDelete = STATUS_CANNOT_DELETE, + IoReparseTagNotHandled = STATUS_IO_REPARSE_TAG_NOT_HANDLED, + DirectoryIsAReparsePoint = STATUS_DIRECTORY_IS_A_REPARSE_POINT, + SharingViolation = STATUS_SHARING_VIOLATION, + DeletePending = STATUS_DELETE_PENDING, + FileSystemVirtualizationInvalidOperation = STATUS_FILE_SYSTEM_VIRTUALIZATION_INVALID_OPERATION, + InsufficientResources = STATUS_INSUFFICIENT_RESOURCES + }; +} \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/Scripts/CreateCliAssemblyVersion.bat b/GVFS/GVFS.GvFltWrapper/Scripts/CreateCliAssemblyVersion.bat new file mode 100644 index 0000000000..2c71b218bc --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/Scripts/CreateCliAssemblyVersion.bat @@ -0,0 +1,4 @@ +mkdir %2\BuildOutput +mkdir %2\BuildOutput\GvFlt +echo #include "stdafx.h" > %2\BuildOutput\GvFlt\AssemblyVersion.h +echo using namespace System::Reflection; [assembly:AssemblyVersion("%1")];[assembly:AssemblyFileVersion("%1")]; >> %2\BuildOutput\GvFlt\AssemblyVersion.h \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/Scripts/CreateVersionHeader.bat b/GVFS/GVFS.GvFltWrapper/Scripts/CreateVersionHeader.bat new file mode 100644 index 0000000000..317ead599c --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/Scripts/CreateVersionHeader.bat @@ -0,0 +1,10 @@ +mkdir %2\BuildOutput +mkdir %2\BuildOutput\GvFlt + +set comma_version_string=%1 +set comma_version_string=%comma_version_string:.=,% + +echo #define GVFLT_FILE_VERSION %comma_version_string% > %2\BuildOutput\GvFlt\VersionHeader.h +echo #define GVFLT_FILE_VERSION_STRING "%1" >> %2\BuildOutput\GvFlt\VersionHeader.h +echo #define GVFLT_PRODUCT_VERSION %comma_version_string% >> %2\BuildOutput\GvFlt\VersionHeader.h +echo #define GVFLT_PRODUCT_VERSION_STRING "%1" >> %2\BuildOutput\GvFlt\VersionHeader.h \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/StatusCode.h b/GVFS/GVFS.GvFltWrapper/StatusCode.h deleted file mode 100644 index 7dc37d65fd..0000000000 --- a/GVFS/GVFS.GvFltWrapper/StatusCode.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -namespace GVFSGvFltWrapper -{ - public enum class StatusCode : long - { - // Subset of NTSTATUS values. Add more values as needed. - StatusSucccess = STATUS_SUCCESS, - StatusTimeout = STATUS_TIMEOUT, - StatusFileNotAvailable = STATUS_FILE_NOT_AVAILABLE, - StatusUnsuccessful = STATUS_UNSUCCESSFUL, - StatusNotImplemented = STATUS_NOT_IMPLEMENTED, - StatusInvalidHandle = STATUS_INVALID_HANDLE, - StatusInvalidParameter = STATUS_INVALID_PARAMETER, - StatusObjectNameNotFound = STATUS_OBJECT_NAME_NOT_FOUND, - StatusObjectPathNotFound = STATUS_OBJECT_PATH_NOT_FOUND, - StatusInvalidDeviceRequest = STATUS_INVALID_DEVICE_REQUEST, - StatusEndOfFile = STATUS_END_OF_FILE, - StatusBufferOverflow = STATUS_BUFFER_OVERFLOW, - StatusInternalError = STATUS_INTERNAL_ERROR, - StatusNoMemory = STATUS_NO_MEMORY, - StatusNoMoreFiles = STATUS_NO_MORE_FILES, - StatusNoSuchFile = STATUS_NO_SUCH_FILE, - StatusRequestAborted = STATUS_REQUEST_ABORTED, - StatusAccessDenied = STATUS_ACCESS_DENIED, - StatusNoInterface = STATUS_NOINTERFACE, - StatusDeviceNotReady = STATUS_DEVICE_NOT_READY, - StatusFileClosed = STATUS_FILE_CLOSED, - StatusObjectNameInvalid = STATUS_OBJECT_NAME_INVALID, - StatusDirectoryNotEmpty = STATUS_DIRECTORY_NOT_EMPTY, - StatusCannotDelete = STATUS_CANNOT_DELETE, - StatusIoReparseTagNotHandled = STATUS_IO_REPARSE_TAG_NOT_HANDLED, - StatusDirectoryIsAReparsePoint = STATUS_DIRECTORY_IS_A_REPARSE_POINT, - StatusSharingViolation = STATUS_SHARING_VIOLATION, - StatusDeletePending = STATUS_DELETE_PENDING, - StatusFileSystemVirtualizationInvalidOperation = STATUS_FILE_SYSTEM_VIRTUALIZATION_INVALID_OPERATION - }; -} \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/Stdafx.cpp b/GVFS/GVFS.GvFltWrapper/Stdafx.cpp index 2e96c7adc6..54abf9f099 100644 --- a/GVFS/GVFS.GvFltWrapper/Stdafx.cpp +++ b/GVFS/GVFS.GvFltWrapper/Stdafx.cpp @@ -1,5 +1,5 @@ // stdafx.cpp : source file that includes just the standard includes -// GVFS.GvFltWrapper.pch will be the pre-compiled header +// GvFlt.pch will be the pre-compiled header // stdafx.obj will contain the pre-compiled type information #include "stdafx.h" \ No newline at end of file diff --git a/GVFS/GVFS.GvFltWrapper/Stdafx.h b/GVFS/GVFS.GvFltWrapper/Stdafx.h index b15ef12c44..260f34977d 100644 --- a/GVFS/GVFS.GvFltWrapper/Stdafx.h +++ b/GVFS/GVFS.GvFltWrapper/Stdafx.h @@ -248,8 +248,8 @@ typedef NTSTATUS(NTAPI *PQueryVolumeInformationFile)(HANDLE, PIO_STATUS_BLOCK, P #include #include #include "gvlib.h" -#include "GvNotificationType.h" -#include "GvUpdateType.h" +#include "NotificationType.h" +#include "UpdateType.h" #include "IoStatusBlockValue.h" diff --git a/GVFS/GVFS.GvFltWrapper/GvUpdateFailureCause.h b/GVFS/GVFS.GvFltWrapper/UpdateFailureCause.h similarity index 74% rename from GVFS/GVFS.GvFltWrapper/GvUpdateFailureCause.h rename to GVFS/GVFS.GvFltWrapper/UpdateFailureCause.h index d97bd841e5..820997db54 100644 --- a/GVFS/GVFS.GvFltWrapper/GvUpdateFailureCause.h +++ b/GVFS/GVFS.GvFltWrapper/UpdateFailureCause.h @@ -1,9 +1,9 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { [System::FlagsAttribute] - public enum class GvUpdateFailureCause : unsigned long + public enum class UpdateFailureCause : unsigned long { NoFailure = GV_UPDATE_FAILURE_CAUSE_NO_FAILURE, DirtyMetadata = GV_UPDATE_FAILURE_CAUSE_DIRTY_METADATA, diff --git a/GVFS/GVFS.GvFltWrapper/UpdateType.h b/GVFS/GVFS.GvFltWrapper/UpdateType.h new file mode 100644 index 0000000000..da2bfccfd6 --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/UpdateType.h @@ -0,0 +1,13 @@ +#pragma once + +namespace GvFlt +{ + [System::FlagsAttribute] + public enum class UpdateType : unsigned long + { + AllowDirtyMetadata = GV_UPDATE_ALLOW_DIRTY_METADATA, + AllowDirtyData = GV_UPDATE_ALLOW_DIRTY_DATA, + AllowTombstone = GV_UPDATE_ALLOW_TOMBSTONE, + AllowReadOnly = GV_UPDATE_ALLOW_READ_ONLY + }; +} diff --git a/GVFS/GVFS.GvFltWrapper/Utils.h b/GVFS/GVFS.GvFltWrapper/Utils.h index 93b5264f43..c2f0a229ad 100644 --- a/GVFS/GVFS.GvFltWrapper/Utils.h +++ b/GVFS/GVFS.GvFltWrapper/Utils.h @@ -1,6 +1,6 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { inline System::Guid GUIDtoGuid(const GUID& guid) { diff --git a/GVFS/GVFS.GvFltWrapper/Version.rc b/GVFS/GVFS.GvFltWrapper/Version.rc index 43dd1f9bd9..cbe1632d4b 100644 Binary files a/GVFS/GVFS.GvFltWrapper/Version.rc and b/GVFS/GVFS.GvFltWrapper/Version.rc differ diff --git a/GVFS/GVFS.GvFltWrapper/GvFltWrapper.cpp b/GVFS/GVFS.GvFltWrapper/VirtualizationInstance.cpp similarity index 64% rename from GVFS/GVFS.GvFltWrapper/GvFltWrapper.cpp rename to GVFS/GVFS.GvFltWrapper/VirtualizationInstance.cpp index c35950fffd..57a304b32f 100644 --- a/GVFS/GVFS.GvFltWrapper/GvFltWrapper.cpp +++ b/GVFS/GVFS.GvFltWrapper/VirtualizationInstance.cpp @@ -1,14 +1,14 @@ #include "stdafx.h" #include "GvFltException.h" -#include "GvFltWrapper.h" -#include "GvDirectoryEnumerationResultImpl.h" -#include "GvDirectoryEnumerationFileNamesResult.h" +#include "VirtualizationInstance.h" +#include "DirectoryEnumerationResultImpl.h" +#include "DirectoryEnumerationFileNamesResult.h" #include "Utils.h" -using namespace GVFS::Common::Tracing; -using namespace GVFSGvFltWrapper; +using namespace GvFlt; using namespace Microsoft::Diagnostics::Tracing; using namespace System; +using namespace System::Collections::Generic; using namespace System::ComponentModel; using namespace System::Text; @@ -16,21 +16,20 @@ namespace { const ULONG READ_BUFFER_SIZE = 64 * 1024; const ULONG IDEAL_WRITE_BUFFER_SIZE = 64 * 1024; - const UCHAR CURRENT_PLACEHOLDER_VERSION = 1; const int EPOCH_RESERVED_BYTES = 4; - ref class ActiveGvFltManager + ref class VirtualizationManager { public: - // Handle to the active GvFltWrapper instance. - // In the future if we support multiple GvFltWrappers per GVFS instance, this can be a map - // of GV_VIRTUALIZATIONINSTANCE_HANDLE to GvFltWrapper. Then in each callback the - // appropriate GvFltWrapper instance can be found (and the callback is delivered). - static GvFltWrapper^ activeGvFltWrapper; + // Handle to the active VirtualizationInstance. + // In the future if we support multiple VirtualizationInstances per provider instance, this can be a map + // of GV_VIRTUALIZATIONINSTANCE_HANDLE to VirtualizationInstance. Then in each callback the + // appropriate VirtualizationInstance instance can be found (and the callback is delivered). + static VirtualizationInstance^ activeInstance = nullptr; }; // GvFlt callback functions that forward the request from GvFlt to the active - // GvFltWrapper (ActiveGvFltManager::activeGvFltWrapper) + // VirtualizationInstance (VirtualizationManager::activeInstance) NTSTATUS GvStartDirectoryEnumerationCB( _In_ GV_VIRTUALIZATIONINSTANCE_HANDLE virtualizationInstanceHandle, _In_ GUID enumerationId, @@ -100,7 +99,7 @@ namespace ); // Internal helper functions used by the above callbacks - GvDirectoryEnumerationResult^ CreateEnumerationResult( + DirectoryEnumerationResult^ CreateEnumerationResult( _In_ FILE_INFORMATION_CLASS fileInformationClass, _In_ PVOID buffer, _In_ ULONG bufferLength, @@ -113,13 +112,9 @@ namespace size_t GetRequiredAlignment(_In_ FILE_INFORMATION_CLASS fileInformationClass); - UCHAR GetPlaceHolderVersion(const GV_PLACEHOLDER_VERSION_INFO& versionInfo); - void SetPlaceHolderVersion(GV_PLACEHOLDER_VERSION_INFO& versionInfo, UCHAR version); + array^ MarshalPlaceholderId(UCHAR* sourceId); - String^ GetContentId(const GV_PLACEHOLDER_VERSION_INFO& versionInfo); - void SetContentId(GV_PLACEHOLDER_VERSION_INFO& versionInfo, String^ contentId); - - void SetEpochId(GV_PLACEHOLDER_VERSION_INFO& versionInfo, String^ epochId); + void CopyPlaceholderId(UCHAR* destinationId, array^ contentId); bool IsPowerOf2(ULONG num); @@ -131,11 +126,11 @@ namespace unsigned long fileAttributes, long long endOfFile, bool directory, - System::String^ contentId, - System::String^ epochId); + array^ contentId, + array^ epochId); } -GvFltWrapper::GvFltWrapper() +VirtualizationInstance::VirtualizationInstance() : virtualizationInstanceHandle(nullptr) , virtualRootPath(nullptr) , writeBufferSize(0) @@ -143,167 +138,167 @@ GvFltWrapper::GvFltWrapper() { } -GvStartDirectoryEnumerationEvent^ GvFltWrapper::OnStartDirectoryEnumeration::get(void) +StartDirectoryEnumerationEvent^ VirtualizationInstance::OnStartDirectoryEnumeration::get(void) { - return this->gvStartDirectoryEnumerationEvent; + return this->startDirectoryEnumerationEvent; } -void GvFltWrapper::OnStartDirectoryEnumeration::set(GvStartDirectoryEnumerationEvent^ eventCB) +void VirtualizationInstance::OnStartDirectoryEnumeration::set(StartDirectoryEnumerationEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvStartDirectoryEnumerationEvent = eventCB; + this->startDirectoryEnumerationEvent = eventCB; } -GvEndDirectoryEnumerationEvent^ GvFltWrapper::OnEndDirectoryEnumeration::get(void) +EndDirectoryEnumerationEvent^ VirtualizationInstance::OnEndDirectoryEnumeration::get(void) { - return this->gvEndDirectoryEnumerationEvent; + return this->endDirectoryEnumerationEvent; } -void GvFltWrapper::OnEndDirectoryEnumeration::set(GvEndDirectoryEnumerationEvent^ eventCB) +void VirtualizationInstance::OnEndDirectoryEnumeration::set(EndDirectoryEnumerationEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvEndDirectoryEnumerationEvent = eventCB; + this->endDirectoryEnumerationEvent = eventCB; } -GvGetDirectoryEnumerationEvent^ GvFltWrapper::OnGetDirectoryEnumeration::get(void) +GetDirectoryEnumerationEvent^ VirtualizationInstance::OnGetDirectoryEnumeration::get(void) { - return this->gvGetDirectoryEnumerationEvent; + return this->getDirectoryEnumerationEvent; } -void GvFltWrapper::OnGetDirectoryEnumeration::set(GvGetDirectoryEnumerationEvent^ eventCB) +void VirtualizationInstance::OnGetDirectoryEnumeration::set(GetDirectoryEnumerationEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvGetDirectoryEnumerationEvent = eventCB; + this->getDirectoryEnumerationEvent = eventCB; } -GvQueryFileNameEvent^ GvFltWrapper::OnQueryFileName::get(void) +QueryFileNameEvent^ VirtualizationInstance::OnQueryFileName::get(void) { - return this->gvQueryFileNameEvent; + return this->queryFileNameEvent; } -void GvFltWrapper::OnQueryFileName::set(GvQueryFileNameEvent^ eventCB) +void VirtualizationInstance::OnQueryFileName::set(QueryFileNameEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvQueryFileNameEvent = eventCB; + this->queryFileNameEvent = eventCB; } -GvGetPlaceHolderInformationEvent^ GvFltWrapper::OnGetPlaceHolderInformation::get(void) +GetPlaceholderInformationEvent^ VirtualizationInstance::OnGetPlaceholderInformation::get(void) { - return this->gvGetPlaceHolderInformationEvent; + return this->getPlaceholderInformationEvent; } -void GvFltWrapper::OnGetPlaceHolderInformation::set(GvGetPlaceHolderInformationEvent^ eventCB) +void VirtualizationInstance::OnGetPlaceholderInformation::set(GetPlaceholderInformationEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvGetPlaceHolderInformationEvent = eventCB; + this->getPlaceholderInformationEvent = eventCB; } -GvGetFileStreamEvent^ GvFltWrapper::OnGetFileStream::get(void) +GetFileStreamEvent^ VirtualizationInstance::OnGetFileStream::get(void) { - return this->gvGetFileStreamEvent; + return this->getFileStreamEvent; } -void GvFltWrapper::OnGetFileStream::set(GvGetFileStreamEvent^ eventCB) +void VirtualizationInstance::OnGetFileStream::set(GetFileStreamEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvGetFileStreamEvent = eventCB; + this->getFileStreamEvent = eventCB; } -GvNotifyFirstWriteEvent^ GvFltWrapper::OnNotifyFirstWrite::get(void) +NotifyFirstWriteEvent^ VirtualizationInstance::OnNotifyFirstWrite::get(void) { - return this->gvNotifyFirstWriteEvent; + return this->notifyFirstWriteEvent; } -void GvFltWrapper::OnNotifyFirstWrite::set(GvNotifyFirstWriteEvent^ eventCB) +void VirtualizationInstance::OnNotifyFirstWrite::set(NotifyFirstWriteEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyFirstWriteEvent = eventCB; + this->notifyFirstWriteEvent = eventCB; } -GvNotifyCreateEvent^ GvFltWrapper::OnNotifyCreate::get(void) +NotifyFileHandleCreatedEvent^ VirtualizationInstance::OnNotifyFileHandleCreated::get(void) { - return this->gvNotifyCreateEvent; + return this->notifyFileHandleCreatedEvent; } -void GvFltWrapper::OnNotifyCreate::set(GvNotifyCreateEvent^ eventCB) +void VirtualizationInstance::OnNotifyFileHandleCreated::set(NotifyFileHandleCreatedEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyCreateEvent = eventCB; + this->notifyFileHandleCreatedEvent = eventCB; } -GvNotifyPreDeleteEvent^ GvFltWrapper::OnNotifyPreDelete::get(void) +NotifyPreDeleteEvent^ VirtualizationInstance::OnNotifyPreDelete::get(void) { - return this->gvNotifyPreDeleteEvent; + return this->notifyPreDeleteEvent; } -void GvFltWrapper::OnNotifyPreDelete::set(GvNotifyPreDeleteEvent^ eventCB) +void VirtualizationInstance::OnNotifyPreDelete::set(NotifyPreDeleteEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyPreDeleteEvent = eventCB; + this->notifyPreDeleteEvent = eventCB; } -GvNotifyPreRenameEvent^ GvFltWrapper::OnNotifyPreRename::get(void) +NotifyPreRenameEvent^ VirtualizationInstance::OnNotifyPreRename::get(void) { - return this->gvNotifyPreRenameEvent; + return this->notifyPreRenameEvent; } -void GvFltWrapper::OnNotifyPreRename::set(GvNotifyPreRenameEvent^ eventCB) +void VirtualizationInstance::OnNotifyPreRename::set(NotifyPreRenameEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyPreRenameEvent = eventCB; + this->notifyPreRenameEvent = eventCB; } -GvNotifyPreSetHardlinkEvent^ GvFltWrapper::OnNotifyPreSetHardlink::get(void) +NotifyPreSetHardlinkEvent^ VirtualizationInstance::OnNotifyPreSetHardlink::get(void) { - return this->gvNotifyPreSetHardlinkEvent; + return this->notifyPreSetHardlinkEvent; } -void GvFltWrapper::OnNotifyPreSetHardlink::set(GvNotifyPreSetHardlinkEvent^ eventCB) +void VirtualizationInstance::OnNotifyPreSetHardlink::set(NotifyPreSetHardlinkEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyPreSetHardlinkEvent = eventCB; + this->notifyPreSetHardlinkEvent = eventCB; } -GvNotifyFileRenamedEvent^ GvFltWrapper::OnNotifyFileRenamed::get(void) +NotifyFileRenamedEvent^ VirtualizationInstance::OnNotifyFileRenamed::get(void) { - return this->gvNotifyFileRenamedEvent; + return this->notifyFileRenamedEvent; } -void GvFltWrapper::OnNotifyFileRenamed::set(GvNotifyFileRenamedEvent^ eventCB) +void VirtualizationInstance::OnNotifyFileRenamed::set(NotifyFileRenamedEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyFileRenamedEvent = eventCB; + this->notifyFileRenamedEvent = eventCB; } -GvNotifyHardlinkCreatedEvent^ GvFltWrapper::OnNotifyHardlinkCreated::get(void) +NotifyHardlinkCreatedEvent^ VirtualizationInstance::OnNotifyHardlinkCreated::get(void) { - return this->gvNotifyHardlinkCreatedEvent; + return this->notifyHardlinkCreatedEvent; } -void GvFltWrapper::OnNotifyHardlinkCreated::set(GvNotifyHardlinkCreatedEvent^ eventCB) +void VirtualizationInstance::OnNotifyHardlinkCreated::set(NotifyHardlinkCreatedEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyHardlinkCreatedEvent = eventCB; + this->notifyHardlinkCreatedEvent = eventCB; } -GvNotifyFileHandleClosedEvent^ GvFltWrapper::OnNotifyFileHandleClosed::get(void) +NotifyFileHandleClosedEvent^ VirtualizationInstance::OnNotifyFileHandleClosed::get(void) { - return this->gvNotifyFileHandleClosedEvent; + return this->notifyFileHandleClosedEvent; } -void GvFltWrapper::OnNotifyFileHandleClosed::set(GvNotifyFileHandleClosedEvent^ eventCB) +void VirtualizationInstance::OnNotifyFileHandleClosed::set(NotifyFileHandleClosedEvent^ eventCB) { this->ConfirmNotStarted(); - this->gvNotifyFileHandleClosedEvent = eventCB; + this->notifyFileHandleClosedEvent = eventCB; } -GVFS::Common::Tracing::ITracer^ GvFltWrapper::Tracer::get(void) +ITracer^ VirtualizationInstance::Tracer::get(void) { return this->tracer; } -HResult GvFltWrapper::GvStartVirtualizationInstance( - GVFS::Common::Tracing::ITracer^ tracerImpl, +HResult VirtualizationInstance::StartVirtualizationInstance( + ITracer^ tracerImpl, System::String^ virtualizationRootPath, unsigned long poolThreadCount, unsigned long concurrentThreadCount) @@ -315,7 +310,7 @@ HResult GvFltWrapper::GvStartVirtualizationInstance( throw gcnew ArgumentNullException(gcnew String("virtualizationRootPath")); } - ActiveGvFltManager::activeGvFltWrapper = this; + VirtualizationManager::activeInstance = this; this->tracer = tracerImpl; this->virtualRootPath = virtualizationRootPath; @@ -345,58 +340,63 @@ HResult GvFltWrapper::GvStartVirtualizationInstance( )); } -HResult GvFltWrapper::GvStopVirtualizationInstance() +HResult VirtualizationInstance::StopVirtualizationInstance() { long result = ::GvStopVirtualizationInstance(this->virtualizationInstanceHandle); if (result == STATUS_SUCCESS) { this->tracer = nullptr; this->virtualizationInstanceHandle = nullptr; - ActiveGvFltManager::activeGvFltWrapper = nullptr; + VirtualizationManager::activeInstance = nullptr; } return static_cast(result); } -HResult GvFltWrapper::GvDetachDriver() +HResult VirtualizationInstance::DetachDriver() { pin_ptr rootPath = PtrToStringChars(this->virtualRootPath); return static_cast(::GvDetachDriver(rootPath)); } -StatusCode GvFltWrapper::GvWriteFile( +NtStatus VirtualizationInstance::WriteFile( Guid streamGuid, - GVFltWriteBuffer^ targetBuffer, + WriteBuffer^ buffer, unsigned long long byteOffset, unsigned long length ) { + if (buffer == nullptr) + { + return NtStatus::InvalidParameter; + } + array^ guidData = streamGuid.ToByteArray(); pin_ptr data = &(guidData[0]); pin_ptr instanceHandle = &(this->virtualizationInstanceHandle); - return static_cast(::GvWriteFile( + return static_cast(::GvWriteFile( *instanceHandle, *(GUID*)data, - targetBuffer->Pointer.ToPointer(), + buffer->Pointer.ToPointer(), byteOffset, length )); } -StatusCode GvFltWrapper::GvDeleteFile(System::String^ relativePath, GvUpdateType updateFlags, GvUpdateFailureCause% failureReason) +NtStatus VirtualizationInstance::DeleteFile(System::String^ relativePath, UpdateType updateFlags, UpdateFailureCause% failureReason) { pin_ptr instanceHandle = &(this->virtualizationInstanceHandle); pin_ptr path = PtrToStringChars(relativePath); ULONG deleteFailureReason = 0; - StatusCode result = static_cast(::GvDeleteFile(*instanceHandle, path, static_cast(updateFlags), &deleteFailureReason)); - failureReason = static_cast(deleteFailureReason); + NtStatus result = static_cast(::GvDeleteFile(*instanceHandle, path, static_cast(updateFlags), &deleteFailureReason)); + failureReason = static_cast(deleteFailureReason); return result; } -StatusCode GvFltWrapper::GvWritePlaceholderInformation( - String^ targetRelPathName, +NtStatus VirtualizationInstance::WritePlaceholderInformation( + String^ relativePath, DateTime creationTime, DateTime lastAccessTime, DateTime lastWriteTime, @@ -404,11 +404,16 @@ StatusCode GvFltWrapper::GvWritePlaceholderInformation( unsigned long fileAttributes, long long endOfFile, bool directory, - String^ contentId, - String^ epochId) + array^ contentId, + array^ epochId) { + if (relativePath == nullptr) + { + return NtStatus::InvalidParameter; + } + pin_ptr instanceHandle = &(this->virtualizationInstanceHandle); - pin_ptr path = PtrToStringChars(targetRelPathName); + pin_ptr path = PtrToStringChars(relativePath); std::shared_ptr fileInformation = CreatePlaceholderInformation( creationTime, lastAccessTime, @@ -420,28 +425,33 @@ StatusCode GvFltWrapper::GvWritePlaceholderInformation( contentId, epochId); - return static_cast(::GvWritePlaceholderInformation( + return static_cast(::GvWritePlaceholderInformation( *instanceHandle, path, fileInformation.get(), FIELD_OFFSET(GV_PLACEHOLDER_INFORMATION, VariableData))); // We have written no variable data } -StatusCode GvFltWrapper::GvCreatePlaceholderAsHardlink( +NtStatus VirtualizationInstance::CreatePlaceholderAsHardlink( System::String^ destinationFileName, System::String^ hardLinkTarget) { + if (destinationFileName == nullptr || hardLinkTarget == nullptr) + { + return NtStatus::InvalidParameter; + } + pin_ptr instanceHandle = &(this->virtualizationInstanceHandle); pin_ptr targetPath = PtrToStringChars(destinationFileName); pin_ptr hardLinkPath = PtrToStringChars(hardLinkTarget); - return static_cast(::GvCreatePlaceholderAsHardlink( + return static_cast(::GvCreatePlaceholderAsHardlink( *instanceHandle, targetPath, hardLinkPath)); } -StatusCode GvFltWrapper::GvUpdatePlaceholderIfNeeded( +NtStatus VirtualizationInstance::UpdatePlaceholderIfNeeded( System::String^ relativePath, System::DateTime creationTime, System::DateTime lastAccessTime, @@ -449,10 +459,10 @@ StatusCode GvFltWrapper::GvUpdatePlaceholderIfNeeded( System::DateTime changeTime, unsigned long fileAttributes, long long endOfFile, - System::String^ contentId, - System::String^ epochId, - GvUpdateType updateFlags, - GvUpdateFailureCause% failureReason) + array^ contentId, + array^ epochId, + UpdateType updateFlags, + UpdateFailureCause% failureReason) { pin_ptr instanceHandle = &(this->virtualizationInstanceHandle); pin_ptr path = PtrToStringChars(relativePath); @@ -468,18 +478,18 @@ StatusCode GvFltWrapper::GvUpdatePlaceholderIfNeeded( epochId); ULONG updateFailureReason = 0; - StatusCode result = static_cast(::GvUpdatePlaceholderIfNeeded( + NtStatus result = static_cast(::GvUpdatePlaceholderIfNeeded( *instanceHandle, path, fileInformation.get(), FIELD_OFFSET(GV_PLACEHOLDER_INFORMATION, VariableData), // We have written no variable data static_cast(updateFlags), &updateFailureReason)); - failureReason = static_cast(updateFailureReason); + failureReason = static_cast(updateFailureReason); return result; } -GvFltWrapper::OnDiskStatus GvFltWrapper::GetFileOnDiskStatus(System::String^ relativePath) +VirtualizationInstance::OnDiskStatus VirtualizationInstance::GetFileOnDiskStatus(System::String^ relativePath) { GUID handleGUID; pin_ptr filePath = PtrToStringChars(relativePath); @@ -490,7 +500,7 @@ GvFltWrapper::OnDiskStatus GvFltWrapper::GetFileOnDiskStatus(System::String^ rel if (!NT_SUCCESS(closeResult)) { - this->Tracer->RelatedError(String::Format("FileExists: GvCloseFile failed for {0}: {1}", relativePath, static_cast(closeResult))); + this->Tracer->TraceError(String::Format("FileExists: GvCloseFile failed for {0}: {1}", relativePath, static_cast(closeResult))); } return OnDiskStatus::Full; @@ -505,12 +515,12 @@ GvFltWrapper::OnDiskStatus GvFltWrapper::GetFileOnDiskStatus(System::String^ rel case STATUS_OBJECT_NAME_NOT_FOUND: return OnDiskStatus::NotOnDisk; default: - throw gcnew GvFltException("ReadFileContents: GvOpenFile failed", static_cast(openResult)); + throw gcnew GvFltException("ReadFileContents: GvOpenFile failed", static_cast(openResult)); break; } } -System::String^ GvFltWrapper::ReadFullFileContents(System::String^ relativePath) +System::String^ VirtualizationInstance::ReadFullFileContents(System::String^ relativePath) { GUID handleGUID; pin_ptr filePath = PtrToStringChars(relativePath); @@ -518,7 +528,7 @@ System::String^ GvFltWrapper::ReadFullFileContents(System::String^ relativePath) if (!NT_SUCCESS(openResult)) { - throw gcnew GvFltException("ReadFileContents: GvOpenFile failed", static_cast(openResult)); + throw gcnew GvFltException("ReadFileContents: GvOpenFile failed", static_cast(openResult)); } StringBuilder^ allLines = gcnew StringBuilder(); @@ -537,10 +547,10 @@ System::String^ GvFltWrapper::ReadFullFileContents(System::String^ relativePath) if (!NT_SUCCESS(closeResult)) { - this->Tracer->RelatedError(String::Format("ReadFileContents: GvCloseFile failed while closing file after failed read of {0}: {1}", relativePath, static_cast(closeResult))); + this->Tracer->TraceError(String::Format("ReadFileContents: GvCloseFile failed while closing file after failed read of {0}: {1}", relativePath, static_cast(closeResult))); } - throw gcnew GvFltException("ReadFileContents: GvReadFile failed", static_cast(readResult)); + throw gcnew GvFltException("ReadFileContents: GvReadFile failed", static_cast(readResult)); } if (bytesRead > 0) @@ -556,36 +566,42 @@ System::String^ GvFltWrapper::ReadFullFileContents(System::String^ relativePath) if (!NT_SUCCESS(closeResult)) { - this->Tracer->RelatedError(String::Format("ReadFileContents: GvCloseFile failed for {0}: {1}", relativePath, static_cast(closeResult))); + this->Tracer->TraceError(String::Format("ReadFileContents: GvCloseFile failed for {0}: {1}", relativePath, static_cast(closeResult))); } return allLines->ToString(); } -ULONG GvFltWrapper::GetWriteBufferSize() +ULONG VirtualizationInstance::GetWriteBufferSize() { return this->writeBufferSize; } -ULONG GvFltWrapper::GetAlignmentRequirement() +ULONG VirtualizationInstance::GetAlignmentRequirement() { return this->alignmentRequirement; } +WriteBuffer^ VirtualizationInstance::CreateWriteBuffer() +{ + return gcnew WriteBuffer( + VirtualizationManager::activeInstance->GetWriteBufferSize(), + VirtualizationManager::activeInstance->GetAlignmentRequirement()); +} + //static -HResult GvFltWrapper::GvConvertDirectoryToVirtualizationRoot(System::Guid virtualizationInstanceGuid, System::String^ rootPathName) +HResult VirtualizationInstance::ConvertDirectoryToVirtualizationRoot(System::Guid virtualizationInstanceGuid, System::String^ rootPath) { array^ guidArray = virtualizationInstanceGuid.ToByteArray(); pin_ptr guidData = &(guidArray[0]); - pin_ptr rootPath = PtrToStringChars(rootPathName); + pin_ptr root = PtrToStringChars(rootPath); GV_PLACEHOLDER_VERSION_INFO versionInfo; memset(&versionInfo, 0, sizeof(GV_PLACEHOLDER_VERSION_INFO)); - SetPlaceHolderVersion(versionInfo, CURRENT_PLACEHOLDER_VERSION); return static_cast(::GvConvertDirectoryToPlaceholder( - rootPath, // RootPathName + root, // RootPathName L"", // TargetPathName &versionInfo, // VersionInfo 0, // ReparseTag @@ -593,15 +609,15 @@ HResult GvFltWrapper::GvConvertDirectoryToVirtualizationRoot(System::Guid virtua *(GUID*)guidData)); // VirtualizationInstanceID } -void GvFltWrapper::ConfirmNotStarted() +void VirtualizationInstance::ConfirmNotStarted() { if (this->virtualizationInstanceHandle) { - throw gcnew GvFltException("Operation invalid after GvFlt is started"); + throw gcnew GvFltException("Operation invalid after virtualization instance is started"); } } -void GvFltWrapper::CalculateWriteBufferSizeAndAlignment() +void VirtualizationInstance::CalculateWriteBufferSizeAndAlignment() { HMODULE ntdll = LoadLibrary(L"ntdll.dll"); if (!ntdll) @@ -704,24 +720,23 @@ void GvFltWrapper::CalculateWriteBufferSizeAndAlignment() if (!IsPowerOf2(this->alignmentRequirement)) { - EventMetadata^ metadata = gcnew EventMetadata(); + Dictionary^ metadata = gcnew Dictionary(); metadata->Add("ErrorMessage", "Failed to determine alignment"); metadata->Add("LogicalBytesPerSector", sectorInfo.LogicalBytesPerSector); metadata->Add("writeBufferSize", this->writeBufferSize); metadata->Add("alignmentRequirement", this->alignmentRequirement); - this->tracer->RelatedError(metadata); + this->tracer->TraceError(metadata); CloseHandle(rootHandle); FreeLibrary(ntdll); throw gcnew GvFltException(String::Format("Failed to determine volume alignment requirement")); } - EventMetadata^ metadata = gcnew EventMetadata(); + Dictionary^ metadata = gcnew Dictionary(); metadata->Add("LogicalBytesPerSector", sectorInfo.LogicalBytesPerSector); metadata->Add("writeBufferSize", this -> writeBufferSize); metadata->Add("alignmentRequirement", this->alignmentRequirement); - - this->tracer->RelatedEvent(EventLevel::Informational, "CalculateWriteBufferSizeAndAlignment", metadata); + this->tracer->TraceEvent(EventLevel::Informational, "CalculateWriteBufferSizeAndAlignment", metadata); CloseHandle(rootHandle); FreeLibrary(ntdll); @@ -739,30 +754,30 @@ namespace UNREFERENCED_PARAMETER(virtualizationInstanceHandle); UNREFERENCED_PARAMETER(versionInfo); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr && - ActiveGvFltManager::activeGvFltWrapper->OnStartDirectoryEnumeration != nullptr) + if (VirtualizationManager::activeInstance != nullptr && + VirtualizationManager::activeInstance->OnStartDirectoryEnumeration != nullptr) { NTSTATUS result = STATUS_SUCCESS; try { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnStartDirectoryEnumeration( + result = static_cast(VirtualizationManager::activeInstance->OnStartDirectoryEnumeration( GUIDtoGuid(enumerationId), gcnew String(pathName))); } catch (GvFltException^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvStartDirectoryEnumerationCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvStartDirectoryEnumerationCB caught GvFltException: " + error->ToString()); result = static_cast(error->ErrorCode); } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvStartDirectoryEnumerationCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvStartDirectoryEnumerationCB caught Win32Exception: " + error->ToString()); result = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvStartDirectoryEnumerationCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvStartDirectoryEnumerationCB fatal exception: " + error->ToString()); throw; } @@ -779,28 +794,28 @@ namespace { UNREFERENCED_PARAMETER(virtualizationInstanceHandle); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr && - ActiveGvFltManager::activeGvFltWrapper->OnEndDirectoryEnumeration != nullptr) + if (VirtualizationManager::activeInstance != nullptr && + VirtualizationManager::activeInstance->OnEndDirectoryEnumeration != nullptr) { NTSTATUS result = STATUS_SUCCESS; try { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnEndDirectoryEnumeration(GUIDtoGuid(enumerationId))); + result = static_cast(VirtualizationManager::activeInstance->OnEndDirectoryEnumeration(GUIDtoGuid(enumerationId))); } catch (GvFltException^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvEndDirectoryEnumerationCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvEndDirectoryEnumerationCB caught GvFltException: " + error->ToString()); result = static_cast(error->ErrorCode); } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvEndDirectoryEnumerationCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvEndDirectoryEnumerationCB caught Win32Exception: " + error->ToString()); result = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvEndDirectoryEnumerationCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvEndDirectoryEnumerationCB fatal exception: " + error->ToString()); throw; } @@ -825,8 +840,8 @@ namespace size_t fileInfoSize = 0; - if (ActiveGvFltManager::activeGvFltWrapper != nullptr && - ActiveGvFltManager::activeGvFltWrapper->OnGetDirectoryEnumeration != nullptr) + if (VirtualizationManager::activeInstance != nullptr && + VirtualizationManager::activeInstance->OnGetDirectoryEnumeration != nullptr) { memset(fileInformation, 0, *length); NTSTATUS resultStatus = STATUS_SUCCESS; @@ -835,8 +850,8 @@ namespace try { PVOID outputBuffer = fileInformation; - GvDirectoryEnumerationResult^ enumerationData = CreateEnumerationResult(fileInformationClass, outputBuffer, *length, fileInfoSize); - StatusCode callbackResult = ActiveGvFltManager::activeGvFltWrapper->OnGetDirectoryEnumeration( + DirectoryEnumerationResult^ enumerationData = CreateEnumerationResult(fileInformationClass, outputBuffer, *length, fileInfoSize); + NtStatus callbackResult = VirtualizationManager::activeInstance->OnGetDirectoryEnumeration( GUIDtoGuid(enumerationId), filterFileName != NULL ? gcnew String(filterFileName) : nullptr, (restartScan != FALSE), @@ -858,19 +873,19 @@ namespace nextEntry = nullptr; } - while (callbackResult == StatusCode::StatusSucccess && nextEntry != nullptr) + while (callbackResult == NtStatus::Succcess && nextEntry != nullptr) { requestedMultipleEntries = true; enumerationData = CreateEnumerationResult(fileInformationClass, nextEntry, static_cast(remainingSpace), fileInfoSize); - callbackResult = ActiveGvFltManager::activeGvFltWrapper->OnGetDirectoryEnumeration( + callbackResult = VirtualizationManager::activeInstance->OnGetDirectoryEnumeration( GUIDtoGuid(enumerationId), filterFileName != NULL ? gcnew String(filterFileName) : nullptr, false, // restartScan enumerationData); - if (callbackResult == StatusCode::StatusSucccess) + if (callbackResult == NtStatus::Succcess) { SetNextEntryOffset(fileInformationClass, previousEntry, static_cast((PUCHAR)nextEntry - (PUCHAR)previousEntry)); @@ -889,19 +904,19 @@ namespace if (requestedMultipleEntries) { - if (callbackResult == StatusCode::StatusBufferOverflow) + if (callbackResult == NtStatus::BufferOverflow) { // We attempted to place multiple entries in the buffer, but not all of them fit, return StatusSucccess // On the next call to GvGetDirectoryEnumerationCB we'll start with the entry that was too // big to fit - callbackResult = StatusCode::StatusSucccess; + callbackResult = NtStatus::Succcess; } - else if (callbackResult == StatusCode::StatusNoMoreFiles) + else if (callbackResult == NtStatus::NoMoreFiles) { // We succeeded in placing all remaining entries in the buffer. Return StatusSucccess to indicate // that there are entries in the buffer. On the next call to GvGetDirectoryEnumerationCB StatusNoMoreFiles // will be returned - callbackResult = StatusCode::StatusSucccess; + callbackResult = NtStatus::Succcess; } } } @@ -910,17 +925,17 @@ namespace } catch (GvFltException^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetDirectoryEnumerationCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetDirectoryEnumerationCB caught GvFltException: " + error->ToString()); resultStatus = static_cast(error->ErrorCode); } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetDirectoryEnumerationCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetDirectoryEnumerationCB caught Win32Exception: " + error->ToString()); resultStatus = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetDirectoryEnumerationCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetDirectoryEnumerationCB fatal exception: " + error->ToString()); throw; } @@ -938,28 +953,28 @@ namespace ) { UNREFERENCED_PARAMETER(virtualizationInstanceHandle); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr && - ActiveGvFltManager::activeGvFltWrapper->OnQueryFileName != nullptr) + if (VirtualizationManager::activeInstance != nullptr && + VirtualizationManager::activeInstance->OnQueryFileName != nullptr) { NTSTATUS result = STATUS_SUCCESS; try { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnQueryFileName(gcnew String(pathFileName))); + result = static_cast(VirtualizationManager::activeInstance->OnQueryFileName(gcnew String(pathFileName))); } catch (GvFltException^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvQueryFileNameCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvQueryFileNameCB caught GvFltException: " + error->ToString()); result = static_cast(error->ErrorCode); } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvQueryFileNameCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvQueryFileNameCB caught Win32Exception: " + error->ToString()); result = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvQueryFileNameCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvQueryFileNameCB fatal exception: " + error->ToString()); throw; } @@ -986,14 +1001,14 @@ namespace UNREFERENCED_PARAMETER(parentDirectoryVersionInfo); UNREFERENCED_PARAMETER(destinationFileName); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr && - ActiveGvFltManager::activeGvFltWrapper->OnGetPlaceHolderInformation != nullptr) + if (VirtualizationManager::activeInstance != nullptr && + VirtualizationManager::activeInstance->OnGetPlaceholderInformation != nullptr) { NTSTATUS result = STATUS_SUCCESS; try { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnGetPlaceHolderInformation( + result = static_cast(VirtualizationManager::activeInstance->OnGetPlaceholderInformation( gcnew String(pathFileName), desiredAccess, shareMode, @@ -1004,17 +1019,17 @@ namespace } catch (GvFltException^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetPlaceholderInformationCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetPlaceholderInformationCB caught GvFltException: " + error->ToString()); result = static_cast(error->ErrorCode); } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetPlaceholderInformationCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetPlaceholderInformationCB caught Win32Exception: " + error->ToString()); result = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetPlaceholderInformationCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetPlaceholderInformationCB fatal exception: " + error->ToString()); throw; } @@ -1039,59 +1054,49 @@ namespace UNREFERENCED_PARAMETER(virtualizationInstanceHandle); UNREFERENCED_PARAMETER(flags); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr) + if (VirtualizationManager::activeInstance != nullptr) { if (versionInfo == NULL) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetFileStreamCB called with null versionInfo, path: " + gcnew String(pathFileName)); - return static_cast(StatusCode::StatusInternalError); - } - else if (GetPlaceHolderVersion(*versionInfo) != CURRENT_PLACEHOLDER_VERSION) - { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError( - "GvGetFileStreamCB: Unexpected placeholder version " + gcnew String(std::to_wstring(GetPlaceHolderVersion(*versionInfo)).c_str()) + " for file " + gcnew String(pathFileName)); - return static_cast(StatusCode::StatusInternalError); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetFileStreamCB called with null versionInfo, path: " + gcnew String(pathFileName)); + return static_cast(NtStatus::InternalError); } - if (ActiveGvFltManager::activeGvFltWrapper->OnGetFileStream != nullptr) + if (VirtualizationManager::activeInstance->OnGetFileStream != nullptr) { NTSTATUS result = STATUS_SUCCESS; try { - GVFltWriteBuffer targetBuffer( - ActiveGvFltManager::activeGvFltWrapper->GetWriteBufferSize(), - ActiveGvFltManager::activeGvFltWrapper->GetAlignmentRequirement()); - - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnGetFileStream( + result = static_cast(VirtualizationManager::activeInstance->OnGetFileStream( gcnew String(pathFileName), byteOffset.QuadPart, length, GUIDtoGuid(streamGuid), - GetContentId(*versionInfo), + MarshalPlaceholderId(versionInfo->ContentID), + MarshalPlaceholderId(versionInfo->EpochID), triggeringProcessId, - triggeringProcessImageFileName != nullptr ? gcnew String(triggeringProcessImageFileName) : System::String::Empty, - %targetBuffer)); + triggeringProcessImageFileName != nullptr ? gcnew String(triggeringProcessImageFileName) : System::String::Empty)); } catch (GvFltException^ error) { switch (error->ErrorCode) { - case StatusCode::StatusFileClosed: + case NtStatus::FileClosed: // StatusFileClosed is expected, and occurs when an application closes a file handle before OnGetFileStream // is complete break; - case StatusCode::StatusObjectNameNotFound: + case NtStatus::ObjectNameNotFound: // GvWriteFile may return STATUS_OBJECT_NAME_NOT_FOUND if the stream guid provided is not valid (doesn’t exist in the stream table). // For each file expansion, GVFlt creates a new get stream session with a new stream guid, the session starts at the beginning of the // file expansion, and ends after the GetFileStream command returns or times out. // - // If we hit this in GVFS, the most common explanation is that we're calling GvWriteFile after the GVFlt thread waiting on the respose - // from GetFileStream has already timed out + // If we hit this in the provider, the most common explanation is that the provider is calling GvWriteFile after the GVFlt thread + // waiting on the respose from GetFileStream has already timed out break; default: - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetFileStreamCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetFileStreamCB caught GvFltException: " + error->ToString()); break; } @@ -1099,12 +1104,12 @@ namespace } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetFileStreamCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetFileStreamCB caught Win32Exception: " + error->ToString()); result = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvGetFileStreamCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvGetFileStreamCB fatal exception: " + error->ToString()); throw; } @@ -1124,29 +1129,31 @@ namespace UNREFERENCED_PARAMETER(virtualizationInstanceHandle); UNREFERENCED_PARAMETER(versionInfo); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr && - ActiveGvFltManager::activeGvFltWrapper->OnNotifyFirstWrite != nullptr) + if (VirtualizationManager::activeInstance != nullptr) { NTSTATUS result = STATUS_SUCCESS; - try - { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnNotifyFirstWrite(gcnew String(pathFileName))); - } - catch (GvFltException^ error) - { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyFirstWriteCB caught GvFltException: " + error->ToString()); - result = static_cast(error->ErrorCode); - } - catch (Win32Exception^ error) - { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyFirstWriteCB caught Win32Exception: " + error->ToString()); - result = Win32ErrorToNtStatus(error->NativeErrorCode); - } - catch (Exception^ error) + if (VirtualizationManager::activeInstance->OnNotifyFirstWrite != nullptr) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyFirstWriteCB fatal exception: " + error->ToString()); - throw; + try + { + result = static_cast(VirtualizationManager::activeInstance->OnNotifyFirstWrite(gcnew String(pathFileName))); + } + catch (GvFltException^ error) + { + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyFirstWriteCB caught GvFltException: " + error->ToString()); + result = static_cast(error->ErrorCode); + } + catch (Win32Exception^ error) + { + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyFirstWriteCB caught Win32Exception: " + error->ToString()); + result = Win32ErrorToNtStatus(error->NativeErrorCode); + } + catch (Exception^ error) + { + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyFirstWriteCB fatal exception: " + error->ToString()); + throw; + } } return result; @@ -1172,7 +1179,7 @@ namespace UNREFERENCED_PARAMETER(streamGuid); UNREFERENCED_PARAMETER(handleGuid); - if (ActiveGvFltManager::activeGvFltWrapper != nullptr) + if (VirtualizationManager::activeInstance != nullptr) { NTSTATUS result = STATUS_SUCCESS; try @@ -1184,9 +1191,9 @@ namespace switch (notificationType) { case GV_NOTIFICATION_POST_CREATE: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyCreate != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyFileHandleCreated != nullptr) { - ActiveGvFltManager::activeGvFltWrapper->OnNotifyCreate( + VirtualizationManager::activeInstance->OnNotifyFileHandleCreated( gcnew String(pathFileName), isDirectory != FALSE, operationParameters->PostCreate.DesiredAccess, @@ -1199,34 +1206,34 @@ namespace break; case GV_NOTIFICATION_PRE_DELETE: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyPreDelete != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyPreDelete != nullptr) { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnNotifyPreDelete(gcnew String(pathFileName), isDirectory != FALSE)); + result = static_cast(VirtualizationManager::activeInstance->OnNotifyPreDelete(gcnew String(pathFileName), isDirectory != FALSE)); } break; case GV_NOTIFICATION_PRE_RENAME: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyPreRename != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyPreRename != nullptr) { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnNotifyPreRename( + result = static_cast(VirtualizationManager::activeInstance->OnNotifyPreRename( gcnew String(pathFileName), gcnew String(destinationFileName))); } break; case GV_NOTIFICATION_PRE_SET_HARDLINK: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyPreSetHardlink != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyPreSetHardlink != nullptr) { - result = static_cast(ActiveGvFltManager::activeGvFltWrapper->OnNotifyPreSetHardlink( + result = static_cast(VirtualizationManager::activeInstance->OnNotifyPreSetHardlink( gcnew String(pathFileName), gcnew String(destinationFileName))); } break; case GV_NOTIFICATION_FILE_RENAMED: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyFileRenamed != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyFileRenamed != nullptr) { - ActiveGvFltManager::activeGvFltWrapper->OnNotifyFileRenamed( + VirtualizationManager::activeInstance->OnNotifyFileRenamed( gcnew String(pathFileName), gcnew String(destinationFileName), isDirectory != FALSE, @@ -1235,18 +1242,18 @@ namespace break; case GV_NOTIFICATION_HARDLINK_CREATED: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyHardlinkCreated != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyHardlinkCreated != nullptr) { - ActiveGvFltManager::activeGvFltWrapper->OnNotifyHardlinkCreated( + VirtualizationManager::activeInstance->OnNotifyHardlinkCreated( gcnew String(pathFileName), gcnew String(destinationFileName)); } break; case GV_NOTIFICATION_FILE_HANDLE_CLOSED: - if (ActiveGvFltManager::activeGvFltWrapper->OnNotifyFileHandleClosed != nullptr) + if (VirtualizationManager::activeInstance->OnNotifyFileHandleClosed != nullptr) { - ActiveGvFltManager::activeGvFltWrapper->OnNotifyFileHandleClosed( + VirtualizationManager::activeInstance->OnNotifyFileHandleClosed( gcnew String(pathFileName), isDirectory != FALSE, (operationParameters->HandleClosed.FileModified != FALSE), @@ -1255,23 +1262,23 @@ namespace break; default: - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyOperationCB unexpected notification type: " + gcnew String(std::to_string(notificationType).c_str())); + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyOperationCB unexpected notification type: " + gcnew String(std::to_string(notificationType).c_str())); break; } } catch (GvFltException^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyOperationCB caught GvFltException: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyOperationCB caught GvFltException: " + error->ToString()); result = static_cast(error->ErrorCode); } catch (Win32Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyOperationCB caught Win32Exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyOperationCB caught Win32Exception: " + error->ToString()); result = Win32ErrorToNtStatus(error->NativeErrorCode); } catch (Exception^ error) { - ActiveGvFltManager::activeGvFltWrapper->Tracer->RelatedError("GvNotifyOperationCB fatal exception: " + error->ToString()); + VirtualizationManager::activeInstance->Tracer->TraceError("GvNotifyOperationCB fatal exception: " + error->ToString()); throw; } @@ -1281,7 +1288,7 @@ namespace return STATUS_INVALID_DEVICE_STATE; } - inline GvDirectoryEnumerationResult^ CreateEnumerationResult( + inline DirectoryEnumerationResult^ CreateEnumerationResult( _In_ FILE_INFORMATION_CLASS fileInformationClass, _In_ PVOID buffer, _In_ ULONG bufferLength, @@ -1291,15 +1298,15 @@ namespace { case FileNamesInformation: fileInfoSize = FIELD_OFFSET(FILE_NAMES_INFORMATION, FileName); - return gcnew GvDirectoryEnumerationFileNamesResult(static_cast(buffer), bufferLength); + return gcnew DirectoryEnumerationFileNamesResult(static_cast(buffer), bufferLength); case FileIdExtdDirectoryInformation: fileInfoSize = FIELD_OFFSET(FILE_ID_EXTD_DIR_INFORMATION, FileName); - return gcnew GvDirectoryEnumerationResultImpl(static_cast(buffer), bufferLength); + return gcnew DirectoryEnumerationResultImpl(static_cast(buffer), bufferLength); case FileIdExtdBothDirectoryInformation: fileInfoSize = FIELD_OFFSET(FILE_ID_EXTD_BOTH_DIR_INFORMATION, FileName); - return gcnew GvDirectoryEnumerationResultImpl(static_cast(buffer), bufferLength); + return gcnew DirectoryEnumerationResultImpl(static_cast(buffer), bufferLength); default: - throw gcnew GvFltException(StatusCode::StatusInvalidDeviceRequest); + throw gcnew GvFltException(NtStatus::InvalidDeviceRequest); } } @@ -1320,7 +1327,7 @@ namespace static_cast(buffer)->NextEntryOffset = offset; break; default: - throw gcnew GvFltException(StatusCode::StatusInvalidDeviceRequest);; + throw gcnew GvFltException(NtStatus::InvalidDeviceRequest);; } } @@ -1335,48 +1342,27 @@ namespace return 4; break; default: - throw gcnew GvFltException(StatusCode::StatusInvalidDeviceRequest);; + throw gcnew GvFltException(NtStatus::InvalidDeviceRequest);; } } - inline UCHAR GetPlaceHolderVersion(const GV_PLACEHOLDER_VERSION_INFO& versionInfo) - { - return versionInfo.EpochID[0]; - } - - inline void SetPlaceHolderVersion(GV_PLACEHOLDER_VERSION_INFO& versionInfo, UCHAR version) - { - // Use the first byte of VersionInfo.EpochID to store GVFS's version number for placeholders - versionInfo.EpochID[0] = version; - } - - inline String^ GetContentId(const GV_PLACEHOLDER_VERSION_INFO& versionInfo) - { - return gcnew String( - static_cast(static_cast(const_cast(versionInfo).ContentID))); - } - - inline void SetContentId(GV_PLACEHOLDER_VERSION_INFO& versionInfo, String^ contentId) + inline array^ MarshalPlaceholderId(UCHAR* sourceId) { - if (contentId->Length > 0) - { - pin_ptr unmangedContentId = PtrToStringChars(contentId); - memcpy( - versionInfo.ContentID, - unmangedContentId, - min(contentId->Length * sizeof(WCHAR), GV_PLACEHOLDER_ID_LENGTH - sizeof(WCHAR))); - } + array^ marshalledId = gcnew array(GV_PLACEHOLDER_ID_LENGTH); + pin_ptr pinnedId = &marshalledId[0]; + memcpy(pinnedId, sourceId, GV_PLACEHOLDER_ID_LENGTH); + return marshalledId; } - inline void SetEpochId(GV_PLACEHOLDER_VERSION_INFO& versionInfo, String^ epochId) + inline void CopyPlaceholderId(UCHAR* destinationId, array^ sourceId) { - if (!String::IsNullOrEmpty(epochId)) + if (sourceId != nullptr && sourceId->Length > 0) { - pin_ptr unmangedEpochId = PtrToStringChars(epochId); + pin_ptr pinnedId = &sourceId[0]; memcpy( - versionInfo.EpochID + EPOCH_RESERVED_BYTES, - unmangedEpochId, - min(epochId->Length * sizeof(WCHAR), (GV_PLACEHOLDER_ID_LENGTH - sizeof(WCHAR)) - sizeof(UCHAR))); + destinationId, + pinnedId, + min(sourceId->Length * sizeof(byte), GV_PLACEHOLDER_ID_LENGTH)); } } @@ -1393,8 +1379,8 @@ namespace unsigned long fileAttributes, long long endOfFile, bool directory, - System::String^ contentId, - System::String^ epochId) + array^ contentId, + array^ epochId) { std::shared_ptr fileInformation(static_cast(malloc(sizeof(GV_PLACEHOLDER_INFORMATION))), free); @@ -1422,9 +1408,8 @@ namespace fileInformation->Flags = 0; memset(&fileInformation->VersionInfo, 0, sizeof(GV_PLACEHOLDER_VERSION_INFO)); - SetPlaceHolderVersion(fileInformation->VersionInfo, CURRENT_PLACEHOLDER_VERSION); - SetEpochId(fileInformation->VersionInfo, epochId); - SetContentId(fileInformation->VersionInfo, contentId); + CopyPlaceholderId(fileInformation->VersionInfo.EpochID, epochId); + CopyPlaceholderId(fileInformation->VersionInfo.ContentID, contentId); return fileInformation; } diff --git a/GVFS/GVFS.GvFltWrapper/VirtualizationInstance.h b/GVFS/GVFS.GvFltWrapper/VirtualizationInstance.h new file mode 100644 index 0000000000..8b76d74488 --- /dev/null +++ b/GVFS/GVFS.GvFltWrapper/VirtualizationInstance.h @@ -0,0 +1,556 @@ +#pragma once + +#include "CallbackDelegates.h" +#include "UpdateFailureCause.h" +#include "HResult.h" +#include "ITracer.h" + +namespace GvFlt +{ + public ref class VirtualizationInstance + { + public: + /// Length of ContentID and EpochID in bytes + static const int PlaceholderIdLength = GV_PLACEHOLDER_ID_LENGTH; + + VirtualizationInstance(); + + /// Start directory enumeration callback + /// + /// This callback is required + property StartDirectoryEnumerationEvent^ OnStartDirectoryEnumeration + { + StartDirectoryEnumerationEvent^ get(void); + void set(StartDirectoryEnumerationEvent^ eventCB); + }; + + /// End directory enumeration callback + /// + /// This callback is required + property EndDirectoryEnumerationEvent^ OnEndDirectoryEnumeration + { + EndDirectoryEnumerationEvent^ get(void); + void set(EndDirectoryEnumerationEvent^ eventCB); + }; + + /// Get next enumeration result callback + /// + /// This callback is required + property GetDirectoryEnumerationEvent^ OnGetDirectoryEnumeration + { + GetDirectoryEnumerationEvent^ get(void); + void set(GetDirectoryEnumerationEvent^ eventCB); + }; + + /// Query file name callback + /// + /// This callback is required + property QueryFileNameEvent^ OnQueryFileName + { + QueryFileNameEvent^ get(void); + void set(QueryFileNameEvent^ eventCB); + } + + /// Get placeholder callback + /// + /// This callback is required + property GetPlaceholderInformationEvent^ OnGetPlaceholderInformation + { + GetPlaceholderInformationEvent^ get(void); + void set(GetPlaceholderInformationEvent^ eventCB); + }; + + /// Get file stream callback + /// + /// This callback is required + property GetFileStreamEvent^ OnGetFileStream + { + GetFileStreamEvent^ get(void); + void set(GetFileStreamEvent^ eventCB); + }; + + /// First write notification callback + /// + /// This callback is optional + property NotifyFirstWriteEvent^ OnNotifyFirstWrite + { + NotifyFirstWriteEvent^ get(void); + void set(NotifyFirstWriteEvent^ eventCB); + }; + + /// File handle created notification callback + /// + /// This callback is optional + property NotifyFileHandleCreatedEvent^ OnNotifyFileHandleCreated + { + NotifyFileHandleCreatedEvent^ get(void); + void set(NotifyFileHandleCreatedEvent^ eventCB); + }; + + /// Pre-delete notification callback + /// + /// This callback is optional + property NotifyPreDeleteEvent^ OnNotifyPreDelete + { + NotifyPreDeleteEvent^ get(void); + void set(NotifyPreDeleteEvent^ eventCB); + } + + /// Pre-rename notification callback + /// + /// This callback is optional + property NotifyPreRenameEvent^ OnNotifyPreRename + { + NotifyPreRenameEvent^ get(void); + void set(NotifyPreRenameEvent^ eventCB); + } + + /// Pre-set-hardlink notification callback + /// + /// This callback is optional + property NotifyPreSetHardlinkEvent^ OnNotifyPreSetHardlink + { + NotifyPreSetHardlinkEvent^ get(void); + void set(NotifyPreSetHardlinkEvent^ eventCB); + } + + /// File renamed notification callback + /// + /// This callback is optional + property NotifyFileRenamedEvent^ OnNotifyFileRenamed + { + NotifyFileRenamedEvent^ get(void); + void set(NotifyFileRenamedEvent^ eventCB); + } + + /// Hardlink created notification callback + /// + /// This callback is optional + property NotifyHardlinkCreatedEvent^ OnNotifyHardlinkCreated + { + NotifyHardlinkCreatedEvent^ get(void); + void set(NotifyHardlinkCreatedEvent^ eventCB); + } + + /// File handle closed notification callback + /// + /// This callback is optional + property NotifyFileHandleClosedEvent^ OnNotifyFileHandleClosed + { + NotifyFileHandleClosedEvent^ get(void); + void set(NotifyFileHandleClosedEvent^ eventCB); + } + + /// Tracer to be used by VirtualizationInstance + /// + /// Cannot be null, an ITracer must be set. + property ITracer^ Tracer + { + ITracer^ get(void); + }; + + /// Starts a GvFlt virtualization instance + /// ITracer implementation used to trace messages from the VirtualizationInstance. Cannot be nullptr. + /// + /// The path to the virtualization root directory. This directory must have already been + /// converted to a virtualization root using ConvertDirectoryToVirtualizationRoot. + /// + /// + /// The number of threads to wait on the completion port and process commands. + /// The PoolThreadCount has to > 4 otherwise an invaid parameter error will be returned. + /// + /// + /// The target maximum number of threads to run concurrently. + /// The actual number of threads can be less than this (if no commands are waiting for threads) + /// or more than this (if one or more threads become computable after waits complete). + /// See also - https://msdn.microsoft.com/en-us/library/windows/desktop/aa363862(v=vs.85).aspx + /// + /// + /// If StartVirtualizationInstance succeeds, Success is returned. + /// + /// If GvFlt filter driver is not loaded, PrivilegeNotHeld is returned. + /// + /// If StartVirtualizationInstance fails, the appropriate error is returned. + /// + /// + /// Currently only one VirtualizationInstance can be running at a time. + /// + /// StartVirtualizationInstance function starts a GvFlt virtualization instance by performing below actions: + /// + /// 1) Attaches the GvFlt driver to the volume that contains the virtualization root + /// 2) Establishes two comm ports to the driver + /// 3) Registers the callback routine that handles commands from GvFlt + /// + /// If GvFlt filter is already attached to the volume, this function can be called from a non-elevated process. + /// Otherwise this function will attempt to attach the filter to the volume which requires admin privilege, + /// access denied error will be returned if called from a non-elevated process. + /// + /// + /// Thrown if provider already has a running virtualization instance + HResult StartVirtualizationInstance( + ITracer^ tracerImpl, + System::String^ virtualizationRootPath, + unsigned long poolThreadCount, + unsigned long concurrentThreadCount); + + /// Stops the virtualization instance + /// If StopVirtualizationInstance succeeds, Success is returned. If StartVirtualizationInstance fails, the appropriate error is returned. + HResult StopVirtualizationInstance(); + + /// Detaches GvFlt driver from the volume + /// If DetachDriver succeeds, Success is returned, otherwise the appropriate error is returned. + /// + /// For this call to succeed, there must not be any active virtualization instance on the volume. + /// All provider processes need to call StopVirtualizationInstance for their active instances first. + /// + HResult DetachDriver(); + + /// Send file stream data to the GvFlt driver in a OnGetFileStream callback + /// + /// The Guid to associate with this file stream for a set of WriteFile commands. + /// The provider mush pass in this Guid when calling WriteFile in the callback for the same file stream. + /// + /// + /// Buffer containing the file stream data. This buffer contains only the file stream data specified + /// by byteOffset and length — no data headers are included. The buffer must be at least “length” bytes in size + /// + /// + /// The offset from the beginning of the file data stream in question to where the data + /// contained in buffer should be written + /// + /// The number of bytes of data that should be written to the file from the supplied buffer + /// + /// If WriteFile succeeds, Success is returned. + /// + /// If buffer is nullptr or Length is 0, InvalidParameter is returned. + /// + /// If WriteFile or GvFlt filter fails to allocate memory, InsufficientResources is returned. + /// + /// If WriteFile fails to send the message to GvFlt filter, InternalError is returned. + /// see also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541513(v=vs.85).aspx + /// + /// If GvFlt responses the message with a unexpected message type, InternalError is returned. + /// + /// If GvFlt filter fails to write to the file, the NtStatus error from file system will be returned. + /// see also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff544610(v=vs.85).aspx + /// + /// OnGetFileStream callback must not return until it completes sending back data with WriteFile + NtStatus WriteFile( + System::Guid streamGuid, + WriteBuffer^ buffer, + unsigned long long byteOffset, + unsigned long length); + + /// Deletes an on-disk file without raising notification callbacks + /// The path (relative to the virtualization root) of the file to delete + /// Any combination of flags from UpdateType + /// + /// [Out] If there is a failure owing to mismatch between file state and the updateFlags used, + /// this contains the reason for the mismatch. + /// + /// + /// If DeleteFile succeeds, Success is returned. + /// + /// If DestinationFileName is NULL or Length is 0, InvalidParameter is returned. + /// + /// If PlaceholderInformation is NULL, InvalidParameter is returned. + /// + /// If the file is a dirty "Placeholder" or a "Partial" file and the flag + /// AllowDirtyMetadata is not one of the set flags, FileSystemVirtualizationInvalidOperation will + /// be returned. FailureReason will indicate the reason as DirtyMetadata. + /// + /// If the file is a "Full" file and flag AllowDirtyData is not one + /// of the set flags FileSystemVirtualizationInvalidOperation + /// will be returned. FailureReason will indicate the reason as DirtyData. + /// + /// If the file is a "Tombstone" and flag AllowTombstone is not one of the set flags + /// FileSystemVirtualizationInvalidOperation will be returned. FailureReason will indicate the reason as Tombstone + /// + /// For all allowed updateFlags values, if the file is a “Virtual File”, GvFlt will not be able to open a handle to it + /// and hence the file system error ObjectNameNotFound would be returned. + /// + /// If the function or GvFlt filter fails to allocate memory, InsufficientResources is returned. + /// + /// If the function fails to send the message to GvFlt filter, InternalError is returned. + /// See also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541513(v=vs.85).aspx + /// + /// If GvFlt responds to the message with a unexpected message type, InternalError is returned. + /// + /// If GvFlt cannot open the file, the NtStatus error from the file system will be returned. + /// + /// For any delete related operation, NtStatus error from the file system will be returned. + /// + /// + /// Based on the value of updateFlags, GvFlt behaves in the following manner : + /// For a file in state : + /// + /// 1. Placeholder / Partial: + /// If metadata is not dirty, GvFlt will always attempt deleting the file + /// irrespective of the flags set in UpdateFlags. + /// + /// If the metadata is dirty, the file will only be deleted if the one of the + /// bits set in updateFlags is AllowDirtyMetadata. + /// + /// 2. Full: + /// This file will be deleted if the one of the bits set in UpdateFlags is AllowDirtyData + /// + /// 3. Tombstone: + /// This file will be deleted if one of the bits set in UpdateFlags is AllowTombstone. + /// + /// Based on the sets of bits set, GvFlt will provide the functionality. The provider + /// can use a combination based on the desired behavior. + /// + /// Note: For a directory, the only valid states are Virtual, Placeholder / Partial with + /// clean and dirty metadata. Hence the flag AllowDirtyMetadata would suffice for a directory. + /// + NtStatus DeleteFile( + System::String^ relativePath, + UpdateType updateFlags, + UpdateFailureCause% failureReason); + + /// Sends file or directory metadata to the GvFlt driver in a GetPlaceholderInformation callback + /// The path (relative to the virtualization root) of the file or folder + /// Creation time + /// Last access time + /// Last write time + /// Change time + /// File attributes + /// File length + /// True if relativePath is a folder, false if relativePath is a file + /// ContentId to store in placeholder, can be nullptr + /// EpochId to store in placeholder, can be nullptr + /// + /// If WritePlaceholderInformation succeeds, Success is returned. + /// + /// If relativePath is nullptr, InvalidParameter is returned. + /// + /// If WritePlaceholderInformation or GvFlt filter fails to allocate memory, InsufficientResources is returned. + /// + /// If WritePlaceholderInformation fails to send the message to GvFlt filter, InternalError is returned. + /// see also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541513(v=vs.85).aspx + /// + /// If GvFlt responds to the message with a unexpected message type, InternalError is returned. + /// + /// If GvFlt filter fails to create the new file, the NtStatus error from file system will be returned. + /// see also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541939(v=vs.85).aspx + /// + /// + /// contentId and epochId have a maximum length of PlaceholderIdLength. Any data beyond + /// PlaceholderIdLength will be ignored. + /// + NtStatus WritePlaceholderInformation( + System::String^ relativePath, + System::DateTime creationTime, + System::DateTime lastAccessTime, + System::DateTime lastWriteTime, + System::DateTime changeTime, + unsigned long fileAttributes, + long long endOfFile, + bool directory, + array^ contentId, + array^ epochId); + + /// Create a hardlink in lieu of creating a placeholder + /// The path (relative to the virtualization root) of the file + /// + /// The full path to the existing file that the placeholder hard link will link to. + /// This path must be on the same volume as the virtualization instance. + /// + /// + /// If CreatePlaceholderAsHardlink succeeds, Success is returned. + /// + /// If destinationFileName or hardLinkTarget is nullptr or empty string, InvalidParameter is returned. + /// + /// If CreatePlaceholderAsHardlink or GvFlt filter fails to allocate memory, InsufficientResources is returned. + /// + /// If CreatePlaceholderAsHardlink fails to send the message to GvFlt filter, InternalError is returned. + /// see also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541513(v=vs.85).aspx + /// + /// If GvFlt responds to the message with a unexpected message type, InternalError is returned. + /// + /// If GvFlt filter fails to create the hard link, the NtStatus error from file system will be returned. + /// see also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541939(v=vs.85).aspx + /// + /// + /// The caller uses CreatePlaceholderAsHardlink to indicate that the placeholder + /// should be a hard link to an already - existing file, instead of a proper placeholder. + /// + NtStatus CreatePlaceholderAsHardlink( + System::String^ destinationFileName, + System::String^ hardLinkTarget); + + /// Update placeholder information for a file + /// The path (relative to the virtualization root) of the file + /// Creation time (to set in placeholder) + /// Last access time (to set in placeholder) + /// Last write time (to set in placeholder) + /// Change time (to set in placeholder) + /// File attributes (to set in placeholder) + /// File length (to set in placeholder) + /// ContentId (to set in placeholder), can be nullptr + /// EpochId (to set in placeholder), can be nullptr + /// Any combination of flags from UpdateType + /// + /// [Out] If there is a failure owing to mismatch between file state and the updateFlags used, + /// this contains the reason for the mismatch. + /// + /// + /// If UpdatePlaceholderIfNeeded succeeds Success is returned. Note that even when no update is required to a + /// Partial or a Placeholder file when its existing content ID matches to the new content ID provided + /// by the provider, Success is returned. + /// + /// If DestinationFileName is NULL or Length is 0, InvalidParameter is returned. + /// + /// If the file is a dirty "Placeholder" or a "Partial" file and the flag AllowDirtyMetadata is not one of + /// the set flags, FileSystemVirtualizationInvalidOperation will be returned. FailureReason + /// will indicate the reason as DirtyMetadata. + /// + /// If the file is a "Full" file and flag AllowDirtyData is not one of the set flags + /// FileSystemVirtualizationInvalidOperation will be returned. FailureReason will + /// indicate the reason as DirtyData. + /// + /// If the file is a "Tombstone" and flag AllowTombstone is not one + /// of the set flags FileSystemVirtualizationInvalidOperation will + /// be returned. FailureReason will indicate the reason as Tombstone. + /// + /// For all allowed UpdateFlags values, if the file is a “Virtual File”, GvFlt will not be able to open + /// a handle to it and hence the file system error ObjectNameNotFound would be returned. + /// + /// If the function or GvFlt filter fails to allocate memory, InsufficientResources is returned. + /// + /// If the function fails to send the message to GvFlt filter, InternalError is returned. + /// See also - https://msdn.microsoft.com/en-us/library/windows/hardware/ff541513(v=vs.85).aspx + /// + /// If GvFlt responds to the message with a unexpected message type, InternalError is returned. + /// + /// If GvFlt cannot open the file, the NtStatus error from the file system will be returned. + /// + /// For all UpdateFlags values the following apply as we follow a Create New, Rename new to old + /// (replace_if_existing set to true) approach: + /// - During the delete operation, errors received from the Filesystem would apply. + /// - During creating a new placeholder, all errors in WritePlaceholderInformation apply. + /// + /// + /// + /// Based on the value of the flags, GvFlt would behave in the following manner- + /// + /// For a file in state : + /// + /// 1. Placeholder / Partial : + /// If metadata is not dirty, GvFlt will always attempt Updating the file + /// irrespective of the flags set in updateFlags. + /// + /// If the metadata is dirty, the file will only be updated if the one of the + /// bits set in UpdateFlags is AllowDirtyMetadata. + /// + /// If the conditions above are satisfied Updates to Partial / Placeholder files + /// will only be allowed if its existing contentIddoes not match the new + /// contentId provided by the provider + /// + /// 2. Full: + /// This file will be updated if the one of the bits set in UpdateFlags is + /// AllowDirtyData. Full File is directly converted to a placeholder + /// with the placeholder values supplied by the Provider. + /// + /// 3. Tombstone : + /// This file will be Updated if one of the bits set in UpdateFlags is + /// AllowTombstone. A tombstone is directly converted to a placeholder + /// with the placeholder values supplied by the Provider. + /// + /// Based on the sets of bits set, GvFlt will provide the functionality. The provider + /// can use a combination based on the desired behavior. + /// + /// Note: For a directory, the only valid states are Virtual, Placeholder / Partial with + /// clean and dirty metadata.Hence the flag AllowDirtyMetadata would suffice for a directory. + /// + /// When no flag is provided or UpdateFlags is set to 0, it can be used for updating a “Placeholder” + /// or a “Partial” file which is in a clean state – i.e. even its metadata is not dirty, with the + /// placeholder information supplied by the provider + /// + NtStatus UpdatePlaceholderIfNeeded( + System::String^ relativePath, + System::DateTime creationTime, + System::DateTime lastAccessTime, + System::DateTime lastWriteTime, + System::DateTime changeTime, + unsigned long fileAttributes, + long long endOfFile, + array^ contentId, + array^ epochId, + UpdateType updateFlags, + UpdateFailureCause% failureReason); + + enum class OnDiskStatus : long + { + NotOnDisk = 0, + Partial = 1, + Full = 2, + OnDiskCannotOpen = 3 + }; + + /// Checks if file is on disk, and whether it's partial or full + /// The path (relative to the virtualization root) of the file + /// OnDiskStatus indicating if the file is not on disk, a partial file, or a full file + /// + /// This function cannot be used to determine if a folder is partial or full, and cannot be + /// used to determine if a path is a file or a folder. + /// + /// Thrown when an error is encountered while trying to open the file + OnDiskStatus GetFileOnDiskStatus(System::String^ relativePath); + + /// Returns the full contents of a file + /// The path (relative to the virtualization root) of the file + /// Contents of the specified full file. BOM, if present, is not removed. + /// Thrown when unable to open the file + System::String^ ReadFullFileContents(System::String^ relativePath); + + /// Create a WriteBuffer (to be used with WriteFile) + /// A newly created WriteBuffer + WriteBuffer^ CreateWriteBuffer(); + + /// Converts an existing folder to a GvFlt virtualization root + /// The Guid that uniquely identifies one virtualization instance + /// Path for the virtualization instance root directory + /// + /// If ConvertDirectoryToVirtualizationRoot succeeds, Ok is returned. + /// + /// If rootPath is a file and not a directory, InvalidArg is returned. + /// + /// If rootPath already contains reparsepoint data, ReparsePointEncountered is returned. + /// + /// If rootPath fails to open, the appropriate error is returned. + /// + static HResult ConvertDirectoryToVirtualizationRoot( + System::Guid virtualizationInstanceGuid, + System::String^ rootPath); + + private: + ULONG GetWriteBufferSize(); + ULONG GetAlignmentRequirement(); + + void ConfirmNotStarted(); + void CalculateWriteBufferSizeAndAlignment(); + + StartDirectoryEnumerationEvent^ startDirectoryEnumerationEvent; + EndDirectoryEnumerationEvent^ endDirectoryEnumerationEvent; + GetDirectoryEnumerationEvent^ getDirectoryEnumerationEvent; + QueryFileNameEvent^ queryFileNameEvent; + GetPlaceholderInformationEvent^ getPlaceholderInformationEvent; + GetFileStreamEvent^ getFileStreamEvent; + NotifyFirstWriteEvent^ notifyFirstWriteEvent; + NotifyFileHandleCreatedEvent^ notifyFileHandleCreatedEvent; + NotifyPreDeleteEvent^ notifyPreDeleteEvent; + NotifyPreRenameEvent^ notifyPreRenameEvent; + NotifyPreSetHardlinkEvent^ notifyPreSetHardlinkEvent; + NotifyFileRenamedEvent^ notifyFileRenamedEvent; + NotifyHardlinkCreatedEvent^ notifyHardlinkCreatedEvent; + NotifyFileHandleClosedEvent^ notifyFileHandleClosedEvent; + + ULONG writeBufferSize; + ULONG alignmentRequirement; + + GV_VIRTUALIZATIONINSTANCE_HANDLE virtualizationInstanceHandle; + System::String^ virtualRootPath; + ITracer^ tracer; + }; +} diff --git a/GVFS/GVFS.GvFltWrapper/GVFltWriteBuffer.cpp b/GVFS/GVFS.GvFltWrapper/WriteBuffer.cpp similarity index 54% rename from GVFS/GVFS.GvFltWrapper/GVFltWriteBuffer.cpp rename to GVFS/GVFS.GvFltWrapper/WriteBuffer.cpp index 3cdad8a024..b2ffaac627 100644 --- a/GVFS/GVFS.GvFltWrapper/GVFltWriteBuffer.cpp +++ b/GVFS/GVFS.GvFltWrapper/WriteBuffer.cpp @@ -1,43 +1,43 @@ #include "stdafx.h" -#include "GvFltWriteBuffer.h" +#include "WriteBuffer.h" using namespace System; using namespace System::IO; -using namespace GVFSGvFltWrapper; +using namespace GvFlt; -GVFltWriteBuffer::GVFltWriteBuffer(ULONG bufferSize, ULONG alignment) +WriteBuffer::WriteBuffer(ULONG bufferSize, ULONG alignment) { this->buffer = (unsigned char*)_aligned_malloc(bufferSize, alignment); if (this->buffer == nullptr) { - throw gcnew InvalidOperationException("Unable to allocate GVFltWriteBuffer"); + throw gcnew InvalidOperationException("Unable to allocate WriteBuffer"); } this->stream = gcnew UnmanagedMemoryStream(buffer, bufferSize, bufferSize, FileAccess::Write); } -GVFltWriteBuffer::~GVFltWriteBuffer() +WriteBuffer::~WriteBuffer() { delete this->stream; - this->!GVFltWriteBuffer(); + this->!WriteBuffer(); } -GVFltWriteBuffer::!GVFltWriteBuffer() +WriteBuffer::!WriteBuffer() { _aligned_free(this->buffer); } -long long GVFltWriteBuffer::Length::get(void) +long long WriteBuffer::Length::get(void) { return this->stream->Length; } -UnmanagedMemoryStream^ GVFltWriteBuffer::Stream::get(void) +UnmanagedMemoryStream^ WriteBuffer::Stream::get(void) { return this->stream; } -IntPtr GVFltWriteBuffer::Pointer::get(void) +IntPtr WriteBuffer::Pointer::get(void) { return IntPtr(this->buffer); } diff --git a/GVFS/GVFS.GvFltWrapper/GVFltWriteBuffer.h b/GVFS/GVFS.GvFltWrapper/WriteBuffer.h similarity index 68% rename from GVFS/GVFS.GvFltWrapper/GVFltWriteBuffer.h rename to GVFS/GVFS.GvFltWrapper/WriteBuffer.h index 43e46fd3cf..989d2f4b2d 100644 --- a/GVFS/GVFS.GvFltWrapper/GVFltWriteBuffer.h +++ b/GVFS/GVFS.GvFltWrapper/WriteBuffer.h @@ -1,12 +1,12 @@ #pragma once -namespace GVFSGvFltWrapper +namespace GvFlt { - public ref class GVFltWriteBuffer + public ref class WriteBuffer { public: - GVFltWriteBuffer(ULONG bufferSize, ULONG alignment); - ~GVFltWriteBuffer(); + WriteBuffer(ULONG bufferSize, ULONG alignment); + ~WriteBuffer(); property long long Length { @@ -24,7 +24,7 @@ namespace GVFSGvFltWrapper } protected: - !GVFltWriteBuffer(); + !WriteBuffer(); private: System::IO::UnmanagedMemoryStream^ stream; diff --git a/GVFS/GVFS.GvFltWrapper/packages.config b/GVFS/GVFS.GvFltWrapper/packages.config index e50eed2fce..f82cab790a 100644 --- a/GVFS/GVFS.GvFltWrapper/packages.config +++ b/GVFS/GVFS.GvFltWrapper/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.csproj index b562d64262..3f66cdc70f 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.csproj @@ -66,9 +66,6 @@ Common\ConsoleHelper.cs - - Common\EnlistmentUtils.cs - Common\GitConfigHelper.cs @@ -81,6 +78,9 @@ Common\GVFSConstants.cs + + Common\GVFSLock.Shared.cs + Common\BrokenPipeException.cs @@ -90,6 +90,9 @@ Common\NamedPipeClient.cs + + Common\Paths.cs + Common\ProcessHelper.cs diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 5c3df00be7..813637b1e5 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -35,7 +35,7 @@ public static void Main(string[] args) ExitWithError("Usage: gvfs.hooks []"); } - enlistmentRoot = EnlistmentUtils.GetEnlistmentRoot(Environment.CurrentDirectory); + enlistmentRoot = Paths.GetGVFSEnlistmentRoot(Environment.CurrentDirectory); if (string.IsNullOrEmpty(enlistmentRoot)) { // Nothing to hook when being run outside of a GVFS repo. @@ -43,7 +43,7 @@ public static void Main(string[] args) Environment.Exit(0); } - enlistmentPipename = NamedPipeClient.GetPipeNameFromPath(enlistmentRoot); + enlistmentPipename = Paths.GetNamedPipeName(enlistmentRoot); switch (GetHookType(args)) { @@ -150,7 +150,7 @@ private static void VerifyRenameDetectionSettings(string[] args) if (!args.Contains("--no-renames") || !args.Contains("--no-breaks")) { Dictionary statusConfig = GitConfigHelper.GetSettings( - Path.Combine(srcRoot, GVFSConstants.DotGit.Config), + File.ReadAllLines(Path.Combine(srcRoot, GVFSConstants.DotGit.Config)), "status"); if (!IsRunningWithParamOrSetting(args, statusConfig, "--no-renames", "renames") || @@ -233,124 +233,59 @@ private static int GetParentPid(string[] args) private static void AcquireGVFSLockForProcess(string fullCommand, int pid, Process parentProcess, NamedPipeClient pipeClient) { - NamedPipeMessages.LockRequest request = - new NamedPipeMessages.LockRequest(pid, fullCommand); - - NamedPipeMessages.Message requestMessage = request.CreateMessage(NamedPipeMessages.AcquireLock.AcquireRequest); - pipeClient.SendRequest(requestMessage); - - NamedPipeMessages.AcquireLock.Response response = new NamedPipeMessages.AcquireLock.Response(pipeClient.ReadResponse()); - - if (response.Result == NamedPipeMessages.AcquireLock.AcceptResult) - { - return; - } - else if (response.Result == NamedPipeMessages.AcquireLock.MountNotReadyResult) - { - ExitWithError("GVFS has not finished initializing, please wait a few seconds and try again."); - } - else + string result; + if (!GVFSLock.TryAcquireGVFSLockForProcess( + pipeClient, + fullCommand, + pid, + parentProcess, + null, // gvfsEnlistmentRoot + out result)) { - string message = string.Empty; - switch (response.Result) - { - case NamedPipeMessages.AcquireLock.AcceptResult: - break; - - case NamedPipeMessages.AcquireLock.DenyGVFSResult: - message = "Waiting for GVFS to release the lock"; - break; - - case NamedPipeMessages.AcquireLock.DenyGitResult: - message = string.Format("Waiting for '{0}' to release the lock", response.ResponseData.ParsedCommand); - break; - - default: - ExitWithError("Error when acquiring the lock. Unrecognized response: " + response.CreateMessage()); - break; - } - - ConsoleHelper.ShowStatusWhileRunning( - () => - { - while (response.Result != NamedPipeMessages.AcquireLock.AcceptResult) - { - Thread.Sleep(250); - pipeClient.SendRequest(requestMessage); - response = new NamedPipeMessages.AcquireLock.Response(pipeClient.ReadResponse()); - } - - return true; - }, - message, - output: Console.Out, - showSpinner: !ConsoleHelper.IsConsoleOutputRedirectedToFile()); + ExitWithError(result); } } private static void ReleaseGVFSLock(string fullCommand, int pid, Process parentProcess, NamedPipeClient pipeClient) { - NamedPipeMessages.LockRequest request = - new NamedPipeMessages.LockRequest(pid, fullCommand); - - NamedPipeMessages.Message requestMessage = request.CreateMessage(NamedPipeMessages.ReleaseLock.Request); - - pipeClient.SendRequest(requestMessage); - NamedPipeMessages.ReleaseLock.Response response = null; - - if (!ConsoleHelper.IsConsoleOutputRedirectedToFile()) - { - // If output is redirected then don't show waiting message or it might be interpreted as error - response = new NamedPipeMessages.ReleaseLock.Response(pipeClient.ReadResponse()); - } - else - { - ConsoleHelper.ShowStatusWhileRunning( - () => - { - response = new NamedPipeMessages.ReleaseLock.Response(pipeClient.ReadResponse()); - - if (response.ResponseData == null) - { - return ConsoleHelper.ActionResult.Failure; - } - - return response.ResponseData.HasFailures ? ConsoleHelper.ActionResult.CompletedWithErrors : ConsoleHelper.ActionResult.Success; - }, - "Waiting for GVFS to parse index and update placeholder files", - output: Console.Out, - showSpinner: true, - suppressGvfsLogMessage: false, - initialDelayMs: PostCommandSpinnerDelayMs); - } - - if (response == null || response.ResponseData == null) - { - Console.WriteLine("\nError communicating with GVFS: Run 'git status' to check the status of your repo"); - } - else if (response.ResponseData.HasFailures) - { - if (response.ResponseData.FailureCountExceedsMaxFileNames) - { - Console.WriteLine( - "\nGVFS failed to update {0} files, run 'git status' to check the status of files in the repo", - response.ResponseData.FailedToDeleteCount + response.ResponseData.FailedToUpdateCount); - } - else + GVFSLock.ReleaseGVFSLock( + pipeClient, + fullCommand, + pid, + parentProcess, + response => { - string deleteFailuresMessage = BuildUpdatePlaceholderFailureMessage(response.ResponseData.FailedToDeleteFileList, "delete", "git clean -f "); - if (deleteFailuresMessage.Length > 0) + if (response == null || response.ResponseData == null) { - Console.WriteLine(deleteFailuresMessage); + Console.WriteLine("\nError communicating with GVFS: Run 'git status' to check the status of your repo"); } - - string updateFailuresMessage = BuildUpdatePlaceholderFailureMessage(response.ResponseData.FailedToUpdateFileList, "update", "git checkout -- "); - if (updateFailuresMessage.Length > 0) + else if (response.ResponseData.HasFailures) { - Console.WriteLine(updateFailuresMessage); + if (response.ResponseData.FailureCountExceedsMaxFileNames) + { + Console.WriteLine( + "\nGVFS failed to update {0} files, run 'git status' to check the status of files in the repo", + response.ResponseData.FailedToDeleteCount + response.ResponseData.FailedToUpdateCount); + } + else + { + string deleteFailuresMessage = BuildUpdatePlaceholderFailureMessage(response.ResponseData.FailedToDeleteFileList, "delete", "git clean -f "); + if (deleteFailuresMessage.Length > 0) + { + Console.WriteLine(deleteFailuresMessage); + } + + string updateFailuresMessage = BuildUpdatePlaceholderFailureMessage(response.ResponseData.FailedToUpdateFileList, "update", "git checkout -- "); + if (updateFailuresMessage.Length > 0) + { + Console.WriteLine(updateFailuresMessage); + } + } } - } - } + }, + gvfsEnlistmentRoot: null, + waitingMessage: "Waiting for GVFS to parse index and update placeholder files", + spinnerDelay: PostCommandSpinnerDelayMs); } private static string BuildUpdatePlaceholderFailureMessage(List fileList, string failedOperation, string recoveryCommand) @@ -437,8 +372,7 @@ private static bool ShouldLock(string[] args) // Don't acquire the lock if we've been explicitly asked not to. This enables tools, such as the VS Git // integration, to provide a "best effort" status without writing to the index. We assume that any such // tools will be constantly polling in the background, so missing a file once isn't a problem. - if (gitCommand == "status" && - args.Contains("--no-lock-index")) + if (gitCommand == "status" && args.Contains("--no-lock-index")) { return false; } diff --git a/GVFS/GVFS.Mount/GVFS.Mount.csproj b/GVFS/GVFS.Mount/GVFS.Mount.csproj index 6703abf029..5134f7a836 100644 --- a/GVFS/GVFS.Mount/GVFS.Mount.csproj +++ b/GVFS/GVFS.Mount/GVFS.Mount.csproj @@ -68,7 +68,6 @@ - @@ -99,7 +98,8 @@ - xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.ReadObjectHook\bin\$(Platform)\$(Configuration)\GVFS.ReadObjectHook.* $(TargetDir) + xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.ReadObjectHook\bin\$(Platform)\$(Configuration)\GVFS.ReadObjectHook.* $(TargetDir) +xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.Hooks\bin\$(Platform)\$(Configuration)\GVFS.Hooks.* $(TargetDir) {17498502-aeff-4e70-90cc-1d0b56a8adf5} GVFS.Mount - - {B8C1DFBA-CAFD-4F7E-A1A3-E11907B5467B} - GVFS.Service - @@ -146,6 +149,7 @@ xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.ReadObjectHook\bin\$(Platform)\$(Configuration)\GVFS.ReadObjectHook.* $(TargetDir) +xcopy /Y $(SolutionDir)..\BuildOutput\GVFS.Hooks\bin\$(Platform)\$(Configuration)\GVFS.Hooks.* $(TargetDir) xcopy /Y $(SolutionDir)..\BuildOutput\GitHooksLoader\bin\$(Platform)\$(Configuration)\GitHooksLoader.* $(TargetDir) cmd /c ""c:\Program Files (x86)\Inno Setup 5\ISCC.exe" "/DPlatformAndConfiguration=$(Platform)\$(Configuration)" /DGvFltPackage="$(GvFltPackage)" $(TargetDir)Setup.iss" diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index b172cdfa47..503a714f93 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -2,6 +2,10 @@ using GVFS.CommandLine; using System; +// This is to keep the reference to GVFS.Mount +// so that the exe will end up in the output directory of GVFS +using GVFS.Mount; + namespace GVFS { public class Program @@ -14,6 +18,7 @@ public static void Main(string[] args) typeof(CloneVerb), // Verbs that require an existing enlistment + typeof(CacheServerVerb), typeof(DehydrateVerb), typeof(DiagnoseVerb), typeof(LogVerb), @@ -35,7 +40,25 @@ public static void Main(string[] args) settings.HelpWriter = Console.Error; }) .ParseArguments(args, verbTypes) - .WithParsed(verb => verb.Execute()); + .WithParsed( + clone => + { + // We handle the clone verb differently, because clone cares if the enlistment path + // was not specified vs if it was specified to be the current directory + clone.Execute(); + }) + .WithParsed( + verb => + { + // For all other verbs, they don't care if the enlistment root is explicitly + // specified or implied to be the current directory + if (string.IsNullOrEmpty(verb.EnlistmentRootPath)) + { + verb.EnlistmentRootPath = Environment.CurrentDirectory; + } + + verb.Execute(); + }); } catch (GVFSVerb.VerbAbortedException e) { diff --git a/GVFS/GVFS/Setup.iss b/GVFS/GVFS/Setup.iss index 8561de03d7..51898d332c 100644 --- a/GVFS/GVFS/Setup.iss +++ b/GVFS/GVFS/Setup.iss @@ -10,6 +10,7 @@ #define MyAppURL "https://github.com/Microsoft/gvfs" #define MyAppExeName "GVFS.exe" #define EnvironmentKey "SYSTEM\CurrentControlSet\Control\Session Manager\Environment" +#define GvFltParametersKey "SYSTEM\CurrentControlSet\Services\Gvflt\Parameters" #define GVFltRelative "..\..\..\..\..\packages\" + GvFltPackage + "\filter" #define GVFSCommonRelative "..\..\..\..\GVFS.Common\bin" @@ -20,9 +21,6 @@ #define GVFSMountRelative "..\..\..\..\GVFS.Mount\bin" #define ReadObjectRelative "..\..\..\..\GVFS.ReadObjectHook\bin" -; Do not use built in InnoSetup constants for .Net location. They do not point to the 64-bit framework -#define InstallUtil "{win}\Microsoft.NET\Framework64\v4.0.30319\installutil.exe" - [Setup] AppId={{489CA581-F131-4C28-BE04-4FB178933E6D} AppName={#MyAppName} @@ -94,7 +92,6 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"Esent.Isam.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"FastFetch.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.Common.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.GVFlt.pdb" -DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.GvFltWrapper.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.pdb" ; GVFS.Service.UI Files @@ -112,7 +109,7 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"Esent.Interop.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"Esent.Isam.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.Common.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.GVFlt.dll" -DestDir: "{app}"; Flags: ignoreversion; Source:"GVFS.GvFltWrapper.dll" +DestDir: "{app}"; Flags: ignoreversion; Source:"GvFlt.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"GvLib.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"Microsoft.Diagnostics.Tracing.EventSource.dll" DestDir: "{app}"; Flags: ignoreversion; Source:"Newtonsoft.Json.dll" @@ -212,7 +209,7 @@ begin try StopGVFSService(); - if not Exec(ExpandConstant('{#InstallUtil}'), '/u ' + ExpandConstant('"{app}\GVFS.Service.exe"'), '', SW_HIDE, ewWaitUntilTerminated, ResultCode) or (ResultCode <> 0) then + if not Exec(ExpandConstant('SC.EXE'), 'delete GVFS.Service', '', SW_HIDE, ewWaitUntilTerminated, ResultCode) or (ResultCode <> 0) then begin RaiseException('Fatal: Could not uninstall existing GVFS.Service.'); end; @@ -245,7 +242,7 @@ begin WizardForm.ProgressGauge.Style := npbstMarquee; try - if Exec(ExpandConstant('{#InstallUtil}'), ExpandConstant('"{app}\GVFS.Service.exe"'), '', SW_HIDE, ewWaitUntilTerminated, ResultCode) and (ResultCode = 0) then + if Exec(ExpandConstant('SC.EXE'), ExpandConstant('create GVFS.Service binPath="{app}\GVFS.Service.exe" start=auto'), '', SW_HIDE, ewWaitUntilTerminated, ResultCode) and (ResultCode = 0) then begin if Exec(ExpandConstant('SC.EXE'), 'failure GVFS.Service reset= 30 actions= restart/10/restart/5000//1', '', SW_HIDE, ewWaitUntilTerminated, ResultCode) then begin @@ -283,9 +280,12 @@ begin Exec(ExpandConstant('SC.EXE'), 'stop gvflt', '', SW_HIDE, ewWaitUntilTerminated, ResultCode); // Note: Programatic install of INF notifies user if the driver being upgraded to is older than the existing, otherwise it works silently... doesn't seem like there is a way to block - if Exec(ExpandConstant('RUNDLL32.EXE'), ExpandConstant('SETUPAPI.DLL,InstallHinfSection DefaultInstall 132 {app}\Filter\gvflt.inf'), '', SW_HIDE, ewWaitUntilTerminated, ResultCode) then + if Exec(ExpandConstant('RUNDLL32.EXE'), ExpandConstant('SETUPAPI.DLL,InstallHinfSection DefaultInstall 128 {app}\Filter\gvflt.inf'), '', SW_HIDE, ewWaitUntilTerminated, ResultCode) then begin - InstallSuccessful := True; + if RegWriteDWordValue(HKEY_LOCAL_MACHINE, '{#GvFltParametersKey}', 'CommandTimeoutInMs', 86400000) then + begin + InstallSuccessful := True; + end; end; finally WizardForm.StatusLabel.Caption := StatusText; @@ -383,4 +383,6 @@ begin begin Abort(); end; + + StopGVFSService(); end; \ No newline at end of file diff --git a/Protocol.md b/Protocol.md index 8c2fa6c019..a172957d58 100644 --- a/Protocol.md +++ b/Protocol.md @@ -175,14 +175,15 @@ will result in a a response like: ``` # `GET /gvfs/config` -This optional endpoint will return all server-set GVFS client configuration options. Its only current -function is to provide an set of allowed GVFS client version ranges as serialized -[System.Version](https://msdn.microsoft.com/en-us/library/system.version\(v=vs.110\).aspx) objects, -in order to block older clients from running in certain scenarios. For example, a data corruption bug -may be found and encouraging clients to avoid that version is desirable. +This optional endpoint will return all server-set GVFS client configuration options. It currently +provides: -In the future, this may be expanded to support other configuration directives interesting to the client, -such as a list of possible cache servers to use. +* A set of allowed GVFS client version ranges, in order to block older clients from running in +certain scenarios. For example, a data corruption bug may be found and encouraging clients to +avoid that version is desirable. +* A list of available cache servers, each describing their url and default-ness with a friendly name +that users can use to inform which cache server to use. Note that the names "None" and "User Defined" +are reserved by GVFS. Any caches with these names may cause undefined behavior in the GVFS client. An example response is provided below. Note that the `null` `"Max"` value is only allowed for the last (or greatest) range, since it logically excludes greater version numbers from having an effect. @@ -232,6 +233,15 @@ An example response is provided below. Note that the `null` `"Max"` value is onl "MajorRevision": 0, "MinorRevision": 1 } + }], + "CacheServers": [{ + "Url": "https://redmond-cache-machine/repo-id", + "Name": "Redmond", + "GlobalDefault": true + }, { + "Url": "https://dublin-cache-machine/repo-id", + "Name": "Dublin", + "GlobalDefault": false }] } ``` diff --git a/Scripts/SetupDevService.bat b/Scripts/SetupDevService.bat new file mode 100644 index 0000000000..317207e3e8 --- /dev/null +++ b/Scripts/SetupDevService.bat @@ -0,0 +1,4 @@ +@ECHO OFF +IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") + +sc create %Configuration%.GVFS.Service binPath=%~dp0\..\..\BuildOutput\GVFS.Service\bin\x64\%Configuration%\GVFS.Service.exe \ No newline at end of file diff --git a/Scripts/StartDevService.bat b/Scripts/StartDevService.bat new file mode 100644 index 0000000000..d3675c0c15 --- /dev/null +++ b/Scripts/StartDevService.bat @@ -0,0 +1,4 @@ +@ECHO OFF +IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") + +sc start %Configuration%.GVFS.Service --servicename=%Configuration%.GVFS.Service \ No newline at end of file diff --git a/Scripts/StopDevService.bat b/Scripts/StopDevService.bat new file mode 100644 index 0000000000..d0d779ad4d --- /dev/null +++ b/Scripts/StopDevService.bat @@ -0,0 +1,4 @@ +@ECHO OFF +IF "%1"=="" (SET "Configuration=Debug") ELSE (SET "Configuration=%1") + +sc stop %Configuration%.GVFS.Service \ No newline at end of file