Skip to content

Commit

Permalink
A better GetAllGlobs (#1871)
Browse files Browse the repository at this point in the history
* A better GetAllGlobs implementation

Resolves #1795

GlobResult needs to offer a globbing object that CPS can match arbitrary strings against.
The glob needed to include all the globs from a project item element's
include attribute, and exclude all the itemspec fragments from the
corresponding Exclude and all subsequent Remove elements that apply to
that include.

This required changing GetAllGlobs such that it incorporates information
from Removes and constructs MSBuildGlobWithGaps objects for each
include operation with globs.

In addition, there should be one glob result aggregating the information per item project element.

* CompositeGlob ctor clones input in ImmutableArray

CompositeGlob needs to be immutable, so entering objects need to be cloned / immutable.
Immutable arrays because they are fast to iterate and don't add a wrapping object.

* ItemSpec knows how to convert itself to IMSBuildGlob
  • Loading branch information
cdmihai committed Mar 22, 2017
1 parent c698d39 commit 1b2c9de
Show file tree
Hide file tree
Showing 11 changed files with 568 additions and 95 deletions.
6 changes: 4 additions & 2 deletions ref/net46/Microsoft.Build/Microsoft.Build.cs
Original file line number Diff line number Diff line change
Expand Up @@ -500,10 +500,12 @@ namespace Microsoft.Build.Evaluation
{
public partial class GlobResult
{
public GlobResult(Microsoft.Build.Construction.ProjectItemElement itemElement, string glob, System.Collections.Generic.IEnumerable<string> excludes) { }
internal GlobResult() { }
public System.Collections.Generic.IEnumerable<string> Excludes { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public string Glob { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public System.Collections.Generic.IEnumerable<string> IncludeGlobs { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public Microsoft.Build.Construction.ProjectItemElement ItemElement { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public Microsoft.Build.Globbing.IMSBuildGlob MsBuildGlob { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
public System.Collections.Generic.IEnumerable<string> Removes { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
}
[System.FlagsAttribute]
public enum NewProjectFileOptions
Expand Down
6 changes: 4 additions & 2 deletions ref/netstandard1.3/Microsoft.Build/Microsoft.Build.cs
Original file line number Diff line number Diff line change
Expand Up @@ -488,10 +488,12 @@ namespace Microsoft.Build.Evaluation
{
public partial class GlobResult
{
public GlobResult(Microsoft.Build.Construction.ProjectItemElement itemElement, string glob, System.Collections.Generic.IEnumerable<string> excludes) { }
internal GlobResult() { }
public System.Collections.Generic.IEnumerable<string> Excludes { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public string Glob { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public System.Collections.Generic.IEnumerable<string> IncludeGlobs { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public Microsoft.Build.Construction.ProjectItemElement ItemElement { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } }
public Microsoft.Build.Globbing.IMSBuildGlob MsBuildGlob { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
public System.Collections.Generic.IEnumerable<string> Removes { [System.Runtime.CompilerServices.CompilerGeneratedAttribute]get { throw null; } [System.Runtime.CompilerServices.CompilerGeneratedAttribute]set { } }
}
[System.FlagsAttribute]
public enum NewProjectFileOptions
Expand Down
148 changes: 114 additions & 34 deletions src/Build.OM.UnitTests/Definition/Project_Tests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@
using Microsoft.Build.Evaluation;
using Microsoft.Build.Execution;
using Microsoft.Build.Framework;
using Microsoft.Build.Globbing;
using Microsoft.Build.Shared;

using Task = System.Threading.Tasks.Task;
// can't use an actual ProvenanceResult because it points to a ProjectItemElement which is hard to mock.
using ProvenanceResultTupleList = System.Collections.Generic.List<System.Tuple<string, Microsoft.Build.Evaluation.Operation, Microsoft.Build.Evaluation.Provenance, int>>;
using GlobResultList = System.Collections.Generic.List<System.Tuple<string, string, System.Collections.Immutable.ImmutableHashSet<string>>>;
using GlobResultList = System.Collections.Generic.List<System.Tuple<string, string[], System.Collections.Immutable.ImmutableHashSet<string>, System.Collections.Immutable.ImmutableHashSet<string>>>;
using InvalidProjectFileException = Microsoft.Build.Exceptions.InvalidProjectFileException;
using ToolLocationHelper = Microsoft.Build.Utilities.ToolLocationHelper;
using TargetDotNetFrameworkVersion = Microsoft.Build.Utilities.TargetDotNetFrameworkVersion;
Expand Down Expand Up @@ -3568,6 +3569,21 @@ public void GetItemProvenanceShouldBeSensitiveToGlobbingCone(string includeGlob,
}
}

[Fact]
public void GetAllGlobsShouldNotFindGlobsIfThereAreNoItemElements()
{
var project =
@"<Project ToolsVersion='msbuilddefaulttoolsversion' DefaultTargets='Build' xmlns='msbuildnamespace'>
<ItemGroup>
</ItemGroup>
</Project>
";

var expected = new GlobResultList();

AssertGlobResult(expected, project);
}

[Fact]
public void GetAllGlobsShouldNotFindGlobsIfThereAreNone()
{
Expand Down Expand Up @@ -3605,55 +3621,112 @@ public void GetAllGlobsShouldNotFindGlobsIfInvokedWithEmptyOrNullArguments()
}

[Fact]
public void GetAllGlobsResultsShouldBeInItemElementOrder()
public void GetAllGlobsShouldFindDirectlyReferencedGlobs()
{
var itemElements = Environment.ProcessorCount * 5;
var expected = new GlobResultList();

var project =
@"<Project ToolsVersion='msbuilddefaulttoolsversion' DefaultTargets='Build' xmlns='msbuildnamespace'>
@"<Project ToolsVersion='msbuilddefaulttoolsversion' DefaultTargets='Build' xmlns='msbuildnamespace'>
<ItemGroup>
{0}
<A Include=`*.a;1;*;2;**;?a` Exclude=`1;*;3`/>
<B Include=`a;b;c` Exclude=`**`/>
</ItemGroup>
</Project>
";
";

var sb = new StringBuilder();
for (int i = 0; i < itemElements; i++)
var expectedIncludes = new[] { "*.a", "*", "**", "?a" };
var expectedExcludes = new[] { "1", "*", "3" }.ToImmutableHashSet();
var expected = new GlobResultList
{
sb.AppendLine($"<i_{i} Include=\"*\"/>");
expected.Add(Tuple.Create($"i_{i}", "*", ImmutableHashSet<string>.Empty));
}

project = string.Format(project, sb);
Tuple.Create("A", expectedIncludes, expectedExcludes, ImmutableHashSet.Create<string>())
};

AssertGlobResult(expected, project);
}

[Fact]
public void GetAllGlobsShouldFindDirectlyReferencedGlobs()
public void GetAllGlobsShouldFindAllExcludesAndRemoves()
{
var project =
@"<Project ToolsVersion='msbuilddefaulttoolsversion' DefaultTargets='Build' xmlns='msbuildnamespace'>
<ItemGroup>
<A Include=`*.a;1;*;2;**;?a` Exclude=`1;*;3`/>
<B Include=`a;b;c` Exclude=`**`/>
<A Include=`*` Exclude=`e*`/>
<A Remove=`a`/>
<A Remove=`b`/>
<A Include=`**` Exclude=`e**`/>
<A Remove=`c`/>
</ItemGroup>
</Project>
";

var expectedExcludes = new[] { "1", "*", "3" }.ToImmutableHashSet();
var expected = new GlobResultList
{
Tuple.Create("A", "*.a", expectedExcludes),
Tuple.Create("A", "*", expectedExcludes),
Tuple.Create("A", "**", expectedExcludes),
Tuple.Create("A", "?a", expectedExcludes)
Tuple.Create("A", new []{"**"}, new [] {"e**"}.ToImmutableHashSet(), new [] {"c"}.ToImmutableHashSet()),
Tuple.Create("A", new []{"*"}, new [] {"e*"}.ToImmutableHashSet(), new [] {"c", "b", "a"}.ToImmutableHashSet()),
};

AssertGlobResult(expected, project);
}

[Theory]
// [InlineData(
// @"
//<A Include=`a;b*;c*;d*;e*;f*` Exclude=`c*;d*`/>
//<A Remove=`e*;f*`/>
//",
// new[] {"ba"},
// new[] {"a", "ca", "da", "ea", "fa"}
// )]
// [InlineData(
// @"
//<A Include=`a;b*;c*;d*;e*;f*` Exclude=`c*;d*`/>
//",
// new[] {"ba", "ea", "fa"},
// new[] {"a", "ca", "da"}
// )]
// [InlineData(
// @"
//<A Include=`a;b*;c*;d*;e*;f*`/>
//",
// new[] {"ba", "ca", "da", "ea", "fa"},
// new[] {"a"}
// )]
[InlineData(
@"
<E Include=`b`/>
<R Include=`c`/>
<A Include=`a*;b*;c*` Exclude=`@(E)`/>
<A Remove=`@(R)`/>
",
new[] {"aa", "bb", "cc"},
new[] {"b", "c"}
)]
public void GetAllGlobsShouldProduceGlobThatMatches(string itemContents, string[] stringsThatShouldMatch, string[] stringsThatShouldNotMatch)
{
var projectTemplate =
@"<Project ToolsVersion='msbuilddefaulttoolsversion' DefaultTargets='Build' xmlns='msbuildnamespace'>
<ItemGroup>
{0}
</ItemGroup>
</Project>
";

var projectContents = string.Format(projectTemplate, itemContents);

var getAllGlobsResult = ObjectModelHelpers.CreateInMemoryProject(projectContents).GetAllGlobs();

var uberGlob = new CompositeGlob(getAllGlobsResult.Select(r => r.MsBuildGlob).ToImmutableArray());

foreach (var matchingString in stringsThatShouldMatch)
{
Assert.True(uberGlob.IsMatch(matchingString));
}

foreach (var nonMatchingString in stringsThatShouldNotMatch)
{
Assert.False(uberGlob.IsMatch(nonMatchingString));
}
}

[Fact]
public void GetAllGlobsShouldFindGlobsByItemType()
{
Expand All @@ -3666,16 +3739,15 @@ public void GetAllGlobsShouldFindGlobsByItemType()
</Project>
";

var expectedIncludes = new[] { "*.a", "*", "**" };
var expectedExcludes = new[] { "1", "*", "3" }.ToImmutableHashSet();
var expected = new GlobResultList
{
Tuple.Create("A", "*.a", expectedExcludes),
Tuple.Create("A", "*", expectedExcludes),
Tuple.Create("A", "**", expectedExcludes)
Tuple.Create("A", expectedIncludes, expectedExcludes, ImmutableHashSet<string>.Empty)
};

AssertGlobResult(expected, project, "A");
AssertGlobResult(new GlobResultList(), project, "NotExistant");
AssertGlobResult(new GlobResultList(), project, "NotExistent");
}

[Fact]
Expand All @@ -3694,7 +3766,7 @@ public void GetAllGlobsShouldFindIndirectlyReferencedGlobsFromProperties()

var expected = new GlobResultList
{
Tuple.Create("A", "*", new[] {"*"}.ToImmutableHashSet()),
Tuple.Create("A", new []{"*"}, new[] {"*"}.ToImmutableHashSet(), ImmutableHashSet<string>.Empty),
};

AssertGlobResult(expected, project);
Expand All @@ -3709,17 +3781,24 @@ public void GetAllGlobsShouldNotFindIndirectlyReferencedGlobsFromItems()
<A Include=`*`/>
<B Include=`@(A)`/>
<C Include=`**` Exclude=`@(A)`/>
<C Remove=`@(A)` />
</ItemGroup>
</Project>
";

var expected = new GlobResultList
using (var testFiles = new Helpers.TestProjectWithFiles(project, new[] { "a", "b" }))
using (var projectCollection = new ProjectCollection())
{
Tuple.Create("A", "*", ImmutableHashSet<string>.Empty),
Tuple.Create("C", "**", ImmutableHashSet<string>.Empty),
};
var globResult = new Project(testFiles.ProjectFile, null, MSBuildConstants.CurrentToolsVersion, projectCollection).GetAllGlobs();

AssertGlobResult(expected, project);
var expected = new GlobResultList
{
Tuple.Create("C", new []{"**"}, new [] {"build.proj", "a", "b"}.ToImmutableHashSet(), new [] {"build.proj", "a", "b"}.ToImmutableHashSet()),
Tuple.Create("A", new []{"*"}, ImmutableHashSet<string>.Empty, ImmutableHashSet<string>.Empty)
};

AssertGlobResultsEqual(expected, globResult);
}
}

private static void AssertGlobResult(GlobResultList expected, string project)
Expand All @@ -3741,8 +3820,9 @@ private static void AssertGlobResultsEqual(GlobResultList expected, List<GlobRes
for (var i = 0; i < expected.Count; i++)
{
Assert.Equal(expected[i].Item1, globs[i].ItemElement.ItemType);
Assert.Equal(expected[i].Item2, globs[i].Glob);
Assert.Equal(expected[i].Item2, globs[i].IncludeGlobs);
Assert.Equal(expected[i].Item3, globs[i].Excludes);
Assert.Equal(expected[i].Item4, globs[i].Removes);
}
}

Expand Down
82 changes: 82 additions & 0 deletions src/Build.UnitTests/Evaluation/ItemSpec_Tests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
//-----------------------------------------------------------------------

using System.Collections.Generic;
using Microsoft.Build.Collections;
using Microsoft.Build.Execution;
using Microsoft.Build.UnitTests.BackEnd;
using Xunit;
using ProjectInstanceItemSpec =
Microsoft.Build.Evaluation.ItemSpec<Microsoft.Build.Execution.ProjectPropertyInstance, Microsoft.Build.Execution.ProjectItemInstance>;
using ProjectInstanceExpander =
Microsoft.Build.Evaluation.Expander<Microsoft.Build.Execution.ProjectPropertyInstance, Microsoft.Build.Execution.ProjectItemInstance>;

namespace Microsoft.Build.UnitTests.OM.Evaluation
{
public class ItemSpec_Tests
{
[Fact]
public void EachFragmentTypeShouldContributeToItemSpecGlob()
{
var itemSpec = CreateItemSpecFrom("a;b*;c*;@(foo)", CreateExpander(new Dictionary<string, string[]> {{"foo", new[] {"d", "e"}}}));

var itemSpecGlob = itemSpec.ToMSBuildGlob();

Assert.True(itemSpecGlob.IsMatch("a"));
Assert.True(itemSpecGlob.IsMatch("bar"));
Assert.True(itemSpecGlob.IsMatch("car"));
Assert.True(itemSpecGlob.IsMatch("d"));
Assert.True(itemSpecGlob.IsMatch("e"));
}

[Fact]
public void FragmentGlobsWorkAfterStateIsPartiallyInitializedByOtherOperations()
{
var itemSpec = CreateItemSpecFrom("a;b*;c*;@(foo)", CreateExpander(new Dictionary<string, string[]> {{"foo", new[] {"d", "e"}}}));

int matches;
// cause partial Lazy state to initialize in the ItemExpressionFragment
itemSpec.FragmentsMatchingItem("e", out matches);

Assert.Equal(1, matches);

var itemSpecGlob = itemSpec.ToMSBuildGlob();

Assert.True(itemSpecGlob.IsMatch("a"));
Assert.True(itemSpecGlob.IsMatch("bar"));
Assert.True(itemSpecGlob.IsMatch("car"));
Assert.True(itemSpecGlob.IsMatch("d"));
Assert.True(itemSpecGlob.IsMatch("e"));
}

private ProjectInstanceItemSpec CreateItemSpecFrom(string itemSpec, ProjectInstanceExpander expander)
{
return new ProjectInstanceItemSpec(itemSpec, expander, MockElementLocation.Instance);
}

private ProjectInstanceExpander CreateExpander(Dictionary<string, string[]> items)
{
var itemDictionary = ToItemDictionary(items);

return new ProjectInstanceExpander(new PropertyDictionary<ProjectPropertyInstance>(), itemDictionary);
}

private static ItemDictionary<ProjectItemInstance> ToItemDictionary(Dictionary<string, string[]> itemTypes)
{
var itemDictionary = new ItemDictionary<ProjectItemInstance>();

var dummyProject = ProjectHelpers.CreateEmptyProjectInstance();

foreach (var itemType in itemTypes)
{
foreach (var item in itemType.Value)
{
itemDictionary.Add(new ProjectItemInstance(dummyProject, itemType.Key, item, dummyProject.FullPath));
}
}

return itemDictionary;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@
<Compile Include="Evaluation\ImportFromMSBuildExtensionsPath_Tests.cs" />
<Compile Include="Evaluation\Preprocessor_Tests.cs" />
<Compile Include="Evaluation\ProjectRootElementCache_Tests.cs" />
<Compile Include="Evaluation\ItemSpec_Tests.cs" />
<Compile Include="Evaluation\ProjectStringCache_Tests.cs" />
<Compile Include="EventArgsFormatting_Tests.cs" />
<Compile Include="ExpressionTreeExpression_Tests.cs" />
Expand Down
Loading

0 comments on commit 1b2c9de

Please sign in to comment.