Skip to content

Commit

Permalink
O(modified)
Browse files Browse the repository at this point in the history 
We now only add files that have actually been modified into the sparse-checkout, meaning git operations no longer get linearly slower as you read more files in the repo.
  • Loading branch information
@sanoursa
sanoursa committed May 23, 2017
1 parent 200ff5a commit 9e25b07
Show file tree
Hide file tree
Showing 141 changed files with 5,336 additions and 2,924 deletions.
1 change: 0 additions & 1 deletion AuthoringTests.md
View file
Expand Up @@ -7,7 +7,6 @@
Our functional tests are in the GVFS.FunctionalTests project. They are built on NUnit 3, which is available as a set of NuGet packages.

To run the functional tests:

1. Open GVFS.sln in Visual Studio
2. Build all, which will download the NUnit framework and runner
3. You have three options for how to run the tests, all of which are equivalent.
Expand Down
1 change: 0 additions & 1 deletion GVFS.sln
View file
Expand Up @@ -8,7 +8,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
.gitattributes = .gitattributes
.gitignore = .gitignore
AuthoringTests.md = AuthoringTests.md
License.md = License.md
nuget.config = nuget.config
Protocol.md = Protocol.md
Readme.md = Readme.md
Expand Down
93 changes: 37 additions & 56 deletions GVFS/FastFetch/CheckoutFetchHelper.cs
View file
@@ -1,24 +1,18 @@
using FastFetch.Git;
using FastFetch.Jobs;
using FastFetch.Jobs.Data;
using GVFS.Common;
using GVFS.Common.Git;
using GVFS.Common.Tracing;
using Microsoft.Diagnostics.Tracing;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace FastFetch
{
public class CheckoutFetchHelper : FetchHelper
{
private const string AreaPath = nameof(CheckoutFetchHelper);

private readonly bool allowIndexMetadataUpdateFromWorkingTree;

private readonly int checkoutThreadCount;

public CheckoutFetchHelper(
Expand Down Expand Up @@ -47,7 +41,7 @@ public override void FastFetch(string branchOrCommit, bool isBranch)
string commitToFetch;
if (isBranch)
{
refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit);
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
Expand All @@ -65,20 +59,20 @@ public override void FastFetch(string branchOrCommit, bool isBranch)
}

this.DownloadMissingCommit(commitToFetch, this.GitObjects);

// Configure pipeline
// Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper like in FetchHelper.cs
// Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs
CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.PathWhitelist, commitToFetch, this.Tracer, this.Enlistment);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects);
FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects);

// Start pipeline
downloader.Start();
blobFinder.Start();
checkout.Start();

blobFinder.WaitForCompletion();
this.HasFailures |= blobFinder.HasFailures;

Expand Down Expand Up @@ -119,57 +113,27 @@ public override void FastFetch(string branchOrCommit, bool isBranch)
}
}

bool indexSigningIsOff = this.GetIsIndexSigningOff();

// Update the index
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational))
EventMetadata updateIndexMetadata = new EventMetadata();
updateIndexMetadata.Add("IndexSigningIsOff", indexSigningIsOff);
using (ITracer activity = this.Tracer.StartActivity("UpdateIndex", EventLevel.Informational, Keywords.Telemetry, updateIndexMetadata))
{
// The first bit of core.gvfs is set if index signing is turned off.
const uint CoreGvfsUnsignedIndexFlag = 1;

GitProcess git = new GitProcess(this.Enlistment);

// Only update the index if index signing is turned off.

// The first bit of core.gvfs is set if signing is turned off.
GitProcess.Result configCoreGvfs = git.GetFromConfig("core.gvfs");
uint valueCoreGvfs;

// No errors getting the configuration *and it is either "true" or numeric with the right bit set.
bool indexSigningIsTurnedOff =
!configCoreGvfs.HasErrors &&
!string.IsNullOrEmpty(configCoreGvfs.Output) &&
(configCoreGvfs.Output.Equals("true", StringComparison.OrdinalIgnoreCase) ||
(uint.TryParse(configCoreGvfs.Output, out valueCoreGvfs) &&
((valueCoreGvfs & CoreGvfsUnsignedIndexFlag) == CoreGvfsUnsignedIndexFlag)));

// Create the index object now so it can track the current index
Index index = indexSigningIsTurnedOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null;

GitProcess.Result result;
using (ITracer updateIndexActivity = this.Tracer.StartActivity("ReadTree", EventLevel.Informational))
{
result = git.ReadTree("HEAD");
}
Index index = indexSigningIsOff ? new Index(this.Enlistment.EnlistmentRoot, activity) : null;

GitIndexGenerator indexGen = new GitIndexGenerator(this.Tracer, this.Enlistment, !indexSigningIsOff);
indexGen.CreateFromHeadTree();
this.HasFailures = indexGen.HasFailures;

if (result.HasErrors)
{
activity.RelatedError("Could not read HEAD tree to update index: " + result.Errors);
this.HasFailures = true;
}
else
if (!indexGen.HasFailures && index != null)
{
if (index != null)
{
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;
// Update from disk only if the caller says it is ok via command line
// or if we updated the whole tree and know that all files are up to date
bool allowIndexMetadataUpdateFromWorkingTree = this.allowIndexMetadataUpdateFromWorkingTree || checkout.UpdatedWholeTree;

// Update
index.UpdateFileSizesAndTimes(allowIndexMetadataUpdateFromWorkingTree);
}
else
{
activity.RelatedEvent(EventLevel.Informational, "Core.gvfs is not set, so not updating index entries with file times and sizes.", null);
}
index.UpdateFileSizesAndTimes(checkout.AddedOrEditedLocalFiles, allowIndexMetadataUpdateFromWorkingTree);
}
}
}
Expand Down Expand Up @@ -200,5 +164,22 @@ protected override void UpdateRefs(string branchOrCommit, bool isBranch, GitRefs

base.UpdateRefs(branchOrCommit, isBranch, refs);
}

private bool GetIsIndexSigningOff()
{
// The first bit of core.gvfs is set if index signing is turned off.
const uint CoreGvfsUnsignedIndexFlag = 1;

GitProcess git = new GitProcess(this.Enlistment);
GitProcess.Result configCoreGvfs = git.GetFromConfig("core.gvfs");
uint valueCoreGvfs;

// No errors getting the configuration and it is either "true" or numeric with the right bit set.
return !configCoreGvfs.HasErrors &&
!string.IsNullOrEmpty(configCoreGvfs.Output) &&
(configCoreGvfs.Output.Equals("true", StringComparison.OrdinalIgnoreCase) ||
(uint.TryParse(configCoreGvfs.Output, out valueCoreGvfs) &&
((valueCoreGvfs & CoreGvfsUnsignedIndexFlag) == CoreGvfsUnsignedIndexFlag)));
}
}
}
2 changes: 2 additions & 0 deletions GVFS/FastFetch/FastFetch.csproj
View file
Expand Up @@ -67,6 +67,8 @@
<Compile Include="FastFetchVerb.cs" />
<Compile Include="CheckoutFetchHelper.cs" />
<Compile Include="FetchHelper.cs" />
<Compile Include="Git\GitIndexGenerator.cs" />
<Compile Include="HashingStream.cs" />
<Compile Include="WorkingTree.cs" />
<Compile Include="GitEnlistment.cs" />
<Compile Include="Git\DiffHelper.cs" />
Expand Down
11 changes: 8 additions & 3 deletions GVFS/FastFetch/FastFetchVerb.cs
View file
Expand Up @@ -197,7 +197,11 @@ private int ExecuteWithExitCode()
{
if (this.Verbose)
{
tracer.AddConsoleEventListener(EventLevel.Informational, Keywords.Any);
tracer.AddDiagnosticConsoleEventListener(EventLevel.Informational, Keywords.Any);
}
else
{
tracer.AddPrettyConsoleEventListener(EventLevel.Error, Keywords.Any);
}

string fastfetchLogFile = Enlistment.GetNewLogFileName(enlistment.FastFetchLogRoot, "fastfetch");
Expand Down Expand Up @@ -249,10 +253,11 @@ private int ExecuteWithExitCode()
doPrefetch,
"Fetching",
output: Console.Out,
showSpinner: !Console.IsOutputRedirected);
showSpinner: !Console.IsOutputRedirected,
suppressGvfsLogMessage: true);

Console.WriteLine();
Console.WriteLine("FastFetch is complete. See the full logs at " + fastfetchLogFile);
Console.WriteLine("See the full log at " + fastfetchLogFile);
}

isSuccess &= !fetchHelper.HasFailures;
Expand Down
17 changes: 9 additions & 8 deletions GVFS/FastFetch/FetchHelper.cs
View file
Expand Up @@ -2,6 +2,7 @@
using FastFetch.Jobs;
using GVFS.Common;
using GVFS.Common.Git;
using GVFS.Common.Http;
using GVFS.Common.Tracing;
using Microsoft.Diagnostics.Tracing;
using System;
Expand All @@ -15,7 +16,7 @@ namespace FastFetch
public class FetchHelper
{
protected readonly Enlistment Enlistment;
protected readonly HttpGitObjects HttpGitObjects;
protected readonly GitObjectsHttpRequestor ObjectRequestor;
protected readonly GitObjects GitObjects;
protected readonly ITracer Tracer;

Expand Down Expand Up @@ -45,8 +46,8 @@ public class FetchHelper
this.ChunkSize = chunkSize;
this.Tracer = tracer;
this.Enlistment = enlistment;
this.HttpGitObjects = new HttpGitObjects(tracer, enlistment, downloadThreadCount);
this.GitObjects = new GitObjects(tracer, enlistment, this.HttpGitObjects);
this.ObjectRequestor = new GitObjectsHttpRequestor(tracer, enlistment, downloadThreadCount);
this.GitObjects = new GitObjects(tracer, enlistment, this.ObjectRequestor);
this.PathWhitelist = new List<string>();

// We never want to update config settings for a GVFSEnlistment
Expand All @@ -55,8 +56,8 @@ public class FetchHelper

public int MaxRetries
{
get { return this.HttpGitObjects.MaxRetries; }
set { this.HttpGitObjects.MaxRetries = value; }
get { return this.ObjectRequestor.MaxRetries; }
set { this.ObjectRequestor.MaxRetries = value; }
}

public bool HasFailures { get; protected set; }
Expand Down Expand Up @@ -105,7 +106,7 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch)
string commitToFetch;
if (isBranch)
{
refs = this.HttpGitObjects.QueryInfoRefs(branchOrCommit);
refs = this.ObjectRequestor.QueryInfoRefs(branchOrCommit);
if (refs == null)
{
throw new FetchException("Could not query info/refs from: {0}", this.Enlistment.RepoUrl);
Expand Down Expand Up @@ -150,7 +151,7 @@ public virtual void FastFetch(string branchOrCommit, bool isBranch)
this.HasFailures |= blobEnumerator.HasFailures;

FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, blobEnumerator.RequiredBlobs, availableBlobs, this.Tracer, this.Enlistment);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.HttpGitObjects, this.GitObjects);
BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.DownloadQueue, availableBlobs, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects);
IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, availableBlobs, this.Tracer, this.GitObjects);

blobFinder.Start();
Expand Down Expand Up @@ -211,7 +212,7 @@ protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects)
startMetadata.Add("CommitSha", commitSha);
startMetadata.Add("CommitDepth", CommitDepth);

using (ITracer activity = this.Tracer.StartActivity("DownloadTrees", EventLevel.Informational, startMetadata))
using (ITracer activity = this.Tracer.StartActivity("DownloadTrees", EventLevel.Informational, Keywords.Telemetry, startMetadata))
{
using (GitCatFileBatchCheckProcess catFileProcess = new GitCatFileBatchCheckProcess(this.Tracer, this.Enlistment))
{
Expand Down
2 changes: 1 addition & 1 deletion GVFS/FastFetch/Git/DiffHelper.cs
View file
Expand Up @@ -82,7 +82,7 @@ public void PerformDiff(string sourceTreeSha, string targetTreeSha)
EventMetadata metadata = new EventMetadata();
metadata.Add("TargetTreeSha", targetTreeSha);
metadata.Add("HeadTreeSha", sourceTreeSha);
using (ITracer activity = this.tracer.StartActivity("PerformDiff", EventLevel.Informational, metadata))
using (ITracer activity = this.tracer.StartActivity("PerformDiff", EventLevel.Informational, Keywords.Telemetry, metadata))
{
GitProcess git = new GitProcess(this.enlistment);

Expand Down

0 comments on commit 9e25b07

Please sign in to comment.