Skip to content

Support build/publish one module via TargetModule property #12664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 21, 2020
Merged

Support build/publish one module via TargetModule property #12664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6eb5021
Support build/publish one module via TargetModule property
dingmeng-xue Aug 13, 2020
5a74dae
Merge remote-tracking branch 'upstream/master' into release
dingmeng-xue Aug 19, 2020
99cc3fe
Suppress warning CS2008
dingmeng-xue Aug 19, 2020
9ee8f3f
Add dummy cs file to walkaroud CS2008
dingmeng-xue Aug 19, 2020
f2a31f9
Add proj authenticators to ignored list
dingmeng-xue Aug 19, 2020
b20bb53
update according to review feedback
dingmeng-xue Aug 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions build.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
/p:Scope
'Subfolder under src': An individual cmdlet module
By default, it builds everything
/p:TargetModule
Just focus on one module and its dependency such as Az.Account. Module name doesn't need to lead with Az.
/p:SkipHelp=True
Skips help generation, mainly for local builds to save time.
-->
Expand Down Expand Up @@ -118,7 +120,7 @@
</FilesChangedTask>

<!-- Get the list of modules changed -->
<FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./ModuleMappings.json">
<FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./ModuleMappings.json" TargetModule="$(TargetModule)">
<Output TaskParameter="Output" ItemName="ModulesChanged" />
</FilterTask>
<Message Importance="high" Text="Filtering help generation and StaticAnalysis by the following modules:" />
Expand All @@ -127,7 +129,7 @@
<Message Importance="high" Text="" />

<!-- Get the list of projects to build -->
<FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./CsprojMappings.json">
<FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./CsprojMappings.json" TargetModule="$(TargetModule)">
<Output TaskParameter="Output" ItemName="ProjectsToBuild" />
</FilterTask>
<Message Importance="high" Text="Filtering projects to build by the following:" />
Expand Down Expand Up @@ -237,16 +239,16 @@

<Target Name="Publish" Condition="'$(Configuration)' == 'Release'">
<Message Importance="high" Text="Publishing Cmdlets using $(Scope) scope" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/NewOutputTypeIndex.ps1 -OutputFile $(RepoArtifacts)/outputtypes.json -BuildConfig $(Configuration)&quot;" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/NewHelpIndex.ps1 -OutputFile $(RepoArtifacts)/index.json -BuildConfig $(Configuration)&quot;" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/NewOutputTypeIndex.ps1 -OutputFile $(RepoArtifacts)/outputtypes.json -BuildConfig $(Configuration)&quot;" Condition="'$(TargetModule)' == ''"/>
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/NewHelpIndex.ps1 -OutputFile $(RepoArtifacts)/index.json -BuildConfig $(Configuration)&quot;" Condition="'$(TargetModule)' == ''"/>

<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CleanupBuild.ps1 -BuildConfig $(Configuration)&quot;" />

<Error Condition="'$(NuGetKey)' == ''" Text="You must provide the NuGetKey parameter to the build: /p:NuGetKey=YOUR_PUBLISHING_KEY" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/PublishModules.ps1 -IsNetCore:$$(NetCore) -BuildConfig $(Configuration) -Scope $(Scope) -ApiKey $(NuGetKey) -RepositoryLocation \&quot;$(NuGetPublishingSource)\&quot;&quot; -NugetExe $(NuGetCommand)" />
</Target>

<Target Name="BuildInstaller" AfterTargets="Publish" Condition="'$(Scope)' == 'All' or '$(Scope)' == 'Latest' or '$(Scope)' == 'Netcore'">
<Target Name="BuildInstaller" AfterTargets="Publish" Condition="('$(Scope)' == 'All' or '$(Scope)' == 'Latest' or '$(Scope)' == 'Netcore') and '$(TargetModule)' == ''">
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. Register-PSRepository -Name MSIcreationrepository -SourceLocation $(RepoArtifacts) -InstallationPolicy Trusted &quot;" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoRoot)/setup/generate.ps1 -repository MSIcreationrepository &quot;" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. Unregister-PSRepository -Name MSIcreationrepository &quot;" />
Expand Down
28 changes: 28 additions & 0 deletions src/StackHCI/Properties/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// ----------------------------------------------------------------------------------
//
// Copyright Microsoft Corporation
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------------------------------------------------------------

using System;
using System.Reflection;
using System.Runtime.InteropServices;

[assembly: AssemblyTitle("Microsoft Azure Powershell - StackHCI")]
[assembly: AssemblyCompany(Microsoft.WindowsAzure.Commands.Common.AzurePowerShell.AssemblyCompany)]
[assembly: AssemblyProduct(Microsoft.WindowsAzure.Commands.Common.AzurePowerShell.AssemblyProduct)]
[assembly: AssemblyCopyright(Microsoft.WindowsAzure.Commands.Common.AzurePowerShell.AssemblyCopyright)]

[assembly: ComVisible(false)]
[assembly: CLSCompliant(false)]
[assembly: Guid("8853A329-2E41-4A6A-808B-9D1170A5BBB4")]
[assembly: AssemblyVersion("0.2.0")]
[assembly: AssemblyFileVersion("0.2.0")]
7 changes: 1 addition & 6 deletions tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/FilesChangedTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public class FilesChangedTask : Task
public string PullRequestNumber { get; set; }

/// <summary>
/// Gets or set the TargetModule, e.g. Az.Storage
/// Gets or set the TargetModule, e.g. Storage
/// </summary>
public string TargetModule { get; set; }

Expand Down Expand Up @@ -137,11 +137,6 @@ public override bool Execute()

FilesChanged = filesChanged.ToArray();
}
else if(!string.IsNullOrEmpty(TargetModule))
{
//Add one FAKE changed file for TargetModule, so TargetModule will be included for FilterTask
FilesChanged = new string[] { $"src/{TargetModule}/changeLog.md" };
}
else
{
FilesChanged = new string[] { };
Expand Down
41 changes: 18 additions & 23 deletions tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/FilterTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ public class FilterTask : Task
[Required]
public string MapFilePath { get; set; }

/// <summary>
/// Gets or set the TargetModule, e.g. Storage
/// </summary>
public string TargetModule { get; set; }

/// <summary>
/// Gets or sets the test assemblies output produced by the task.
/// </summary>
Expand All @@ -64,31 +69,26 @@ public override bool Execute()
throw new FileNotFoundException("The MapFilePath provided could not be found. Please provide a valid MapFilePath.");
}

var debugEnvironmentVariable = Environment.GetEnvironmentVariable("DebugLocalBuildTasks");
bool debug;
if (!Boolean.TryParse(debugEnvironmentVariable, out debug))
{
debug = false;
}
var mappingsDictionary = JsonConvert.DeserializeObject<Dictionary<string, string[]>>(File.ReadAllText(MapFilePath));

if (FilesChanged != null && FilesChanged.Length > 0)
{
Output = GetOutput(FilesChanged, MapFilePath);
Console.WriteLine($"Filter according to {FilesChanged.Length} file(s) in FilesChanged");
var filesChangedSet = new HashSet<string>(FilesChanged);
Output = SetGenerator.Generate(filesChangedSet, mappingsDictionary).ToArray();
}
else if(!string.IsNullOrWhiteSpace(TargetModule))
{
Console.WriteLine($"Filter module {TargetModule}");
var modules = (TargetModule.Equals("Accounts"))? new string[] { "Accounts"} : new string[] { "Accounts" , TargetModule};
Output = SetGenerator.Generate(modules, mappingsDictionary).ToArray();
}
else
{
if (debug)
{
Console.WriteLine("Debug: ModuleMapping.json");
var lines = File.ReadAllLines(MapFilePath);
foreach (var line in lines)
{
Console.WriteLine(line);
}
}
Console.WriteLine($"Skip filter and load all from ${MapFilePath}");
var set = new HashSet<string>();
var dictionary = JsonConvert.DeserializeObject<Dictionary<string, string[]>>(File.ReadAllText(MapFilePath));
foreach (KeyValuePair<string, string[]> pair in dictionary)
mappingsDictionary = JsonConvert.DeserializeObject<Dictionary<string, string[]>>(File.ReadAllText(MapFilePath));
foreach (KeyValuePair<string, string[]> pair in mappingsDictionary)
{
set.UnionWith(pair.Value);
}
Expand All @@ -98,10 +98,5 @@ public override bool Execute()

return true;
}

public string[] GetOutput(string[] filesChanged, string mapFilePath)
{
return SetGenerator.Generate(filesChanged, mapFilePath).ToArray();
}
}
}
62 changes: 29 additions & 33 deletions tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/SetGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ namespace Microsoft.WindowsAzure.Build.Tasks
{
using System;
using System.Collections.Generic;
using Newtonsoft.Json;
using System.IO;
using System.Linq;

/// <summary>
Expand All @@ -32,36 +30,6 @@ public static class SetGenerator
/// </summary>
public const int MaxFilesPossible = 300;

/// <summary>
/// Static method used to generate a set of tests to be run based on
/// a Json file which maps files to test Dlls.
/// </summary>
/// <param name="files">This is a set of paths.</param>
/// <param name="mapFilePath">This is the filepath of the map that contains
/// the mapping between files and test DLLs.</param>
/// <returns>Set of tests to be run</returns>
public static IEnumerable<string> Generate(IEnumerable<string> filesChanged, string mapFilePath)
{
if (mapFilePath == null)
{
throw new ArgumentNullException("The mappings file path cannot be null.");
}

if (!File.Exists(mapFilePath))
{
throw new FileNotFoundException("The file path provided for the mappings could not be found.");
}

if (filesChanged == null)
{
throw new ArgumentNullException("The list of files changed cannot be null.");
}

var filesChangedSet = new HashSet<string>(filesChanged);
var mappingsDictionary = JsonConvert.DeserializeObject<Dictionary<string, string[]>>(File.ReadAllText(mapFilePath));

return SetGenerator.Generate(filesChangedSet, mappingsDictionary);
}
/// <summary>
/// Static method used to generate a set of tests to be run based on
/// a set of paths
Expand Down Expand Up @@ -89,7 +57,6 @@ public static HashSet<string> Generate(HashSet<string> filesChangedSet, Dictiona
}

var outputSet = new HashSet<string>();
var filesProvidedCount = filesChangedSet.Count;
var filesFoundCount = 0;
var useFullMapping = false;
if (filesChangedSet.Count >= MaxFilesPossible || filesChangedSet.Count == 0)
Expand Down Expand Up @@ -152,5 +119,34 @@ public static HashSet<string> Generate(HashSet<string> filesChangedSet, Dictiona

return outputSet;
}
/// <summary>
/// Generate set according to module names
/// </summary>
/// <param name="moduleNames"></param>
/// <param name="mappingsDictionary"></param>
/// <returns></returns>
public static HashSet<string> Generate(string[] moduleNames, Dictionary<string, string[]> mappingsDictionary)
{
var outputSet = new HashSet<string>();

foreach (var module in moduleNames)
{
var key = $"src/{module}/";
if (mappingsDictionary.ContainsKey(key))
{
var lines = mappingsDictionary[key];
foreach (var line in lines)
{
if (line.Contains($"/src/{module}/") || line.Contains($"\\src\\{module}\\") || line.Equals($"Az.{module}"))
{
outputSet.Add(line);
}
}
}
}

return outputSet;
}

}
}
4 changes: 2 additions & 2 deletions tools/CreateFilterMappings.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ function Add-ProjectDependencies
[string]$SolutionPath
)

$CommonProjectsToIgnore = @( "Authentication", "Authentication.ResourceManager", "Authenticators", "ScenarioTest.ResourceManager", "TestFx", "Tests" )
$CommonProjectsToIgnore = @("Authenticators", "ScenarioTest.ResourceManager", "TestFx", "Tests" )

$ProjectDependencies = @()
$Content = Get-Content -Path $SolutionPath
Expand Down Expand Up @@ -220,7 +220,7 @@ function Create-ModuleMappings
$Script:ModuleMappings = Initialize-Mappings -PathsToIgnore $PathsToIgnore -CustomMappings $CustomMappings
foreach ($ServiceFolder in $Script:ServiceFolders)
{
$Key = "src/$($ServiceFolder.Name)"
$Key = "src/$($ServiceFolder.Name)/"
$ModuleManifestFiles = Get-ChildItem -Path $ServiceFolder.FullName -Filter "*.psd1" -Recurse | Where-Object { $_.FullName -notlike "*.Test*" -and `
$_.FullName -notlike "*Release*" -and `
$_.FullName -notlike "*Debug*" -and `
Expand Down
6 changes: 6 additions & 0 deletions tools/PublishModules.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,12 @@ function Get-AllModules {
Write-Host "Getting Azure client modules"
$clientModules = Get-ClientModules -BuildConfig $BuildConfig -Scope $Scope -PublishLocal:$PublishLocal -IsNetCore:$isNetCore
Write-Host " "

if($clientModules.Length -le 2) {
return @{
ClientModules = $clientModules
}
}

Write-Host "Getting admin modules"
$adminModules = Get-AdminModules -BuildConfig $BuildConfig -Scope $Scope
Expand Down