Feature/2022 10 updates (#153)

* correct the gh workflow * update wf secret * update nuget dependencies * readme addition * fix SemVersion method deprecation * nuget updates * .net 6 Parallel.ForEachAsync additions * AzurePipelinesToGitHubActionsConverter nuget updated * added example commandLineArgs for local testing * dependabot tweak * more dependabot ignores * add Deploy Phase Type filter Co-authored-by: Azure DevOps CI <agent@dev.azure.com>
f2calv · Oct 8, 2022 · 0ccddb9 · 0ccddb9
1 parent 429a52c
commit 0ccddb9
Show file tree

Hide file tree

Showing 9 changed files with 80 additions and 29 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -7,9 +7,13 @@ updates:
     time: "04:00"
   open-pull-requests-limit: 10
   ignore:
-  - dependency-name: AzurePipelinesToGitHubActionsConverter.Core
   - dependency-name: YamlDotNet
   - dependency-name: CasCap.Common.Extensions
   - dependency-name: CasCap.Common.Net
   - dependency-name: CasCap.Common.Threading
+  - dependency-name: CasCap.Common.Testing
   - dependency-name: Microsoft.NET.Test.Sdk
+  - dependency-name: xunit
+  - dependency-name: xunit.runner.visualstudio
+  - dependency-name: semver
+  - dependency-name: ShellProgressBar
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,8 +9,8 @@ on:
         required: true
         default: Release
         options:
-        - Release
-        - Debug
+          - Release
+          - Debug
       PublishPreview:
         type: string
         description: Publish preview branch?
@@ -32,4 +32,6 @@ jobs:
     uses: f2calv/gha-workflows/.github/workflows/dotnet-publish-nuget-v1.yml@main
     with:
       BuildConfiguration: ${{ github.event.inputs.BuildConfiguration }}
-      PublishPreview: ${{ github.event.inputs.PublishPreview }}
+      PublishPreview: ${{ github.event.inputs.PublishPreview }}
+    secrets:
+      NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}
diff --git a/README.md b/README.md
@@ -17,6 +17,8 @@ Also... there is an optional switch whereby the tool can pass the Azure DevOps P
 
 **Disclaimer**: _Do not consider any of the YAML generated by this tool to be 'production ready'. Do your own testing/research and [post any issues](https://github.com/f2calv/yamlizr/issues) and/or make a PR!_
 
+If you find this tool of use then please give it a thumbs-up by giving this repository a :star: ... :wink:
+
 ## Installation/Set-up
 
 - [Create a Personal Access Token (PAT)](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page) with the following scopes/permissions;
@@ -45,11 +47,15 @@ For context-sensitive help execute;
 yamlizr --help
 ```
 
+Optional arguments;
+
+- `--filter <some string here>` filter build/release definitions (if you want to use a more granular approach).
+- `--phasetype <phase type here>` filter deployment jobs by Deploy Phase Type the default is `AgentBasedDeployment` DeployPhaseTypes(tested), other (un-tested) options are `RunOnServer`, `MachineGroupBasedDeployment` & `DeploymentGates`.
+
 Optional switches;
 
 - `--inline` merge the tasks from task groups into the steps of the calling job instead of creating additional template files.
 - `--githubactions` generate GitHub Actions workflows via [AzurePipelinesToGitHubActionsConverter](https://github.com/samsmithnz/AzurePipelinesToGitHubActionsConverter).
-- `--filter` filter build/release definitions (if you want to use a more granular approach).
 
 To generate both Azure Pipelines and GitHub Actions YAML for a build definition called 'wibble-CI' and a release definition called 'wibble-CD';
 

diff --git a/src/CasCap.Apis.AzureDevOps.Tests/CasCap.Apis.AzureDevOps.Tests.csproj b/src/CasCap.Apis.AzureDevOps.Tests/CasCap.Apis.AzureDevOps.Tests.csproj
@@ -10,14 +10,14 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="CasCap.Common.Testing" Version="1.2.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="CasCap.Common.Testing" Version="1.2.7" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="6.0.1" />
-    <PackageReference Include="xunit" Version="2.4.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+    <PackageReference Include="xunit" Version="2.4.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

diff --git a/src/CasCap.Apis.AzureDevOps/CasCap.Apis.AzureDevOps.csproj b/src/CasCap.Apis.AzureDevOps/CasCap.Apis.AzureDevOps.csproj
@@ -13,17 +13,16 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AzurePipelinesToGitHubActionsConverter.Core" Version="0.21.16" />
-    <PackageReference Include="CasCap.Common.Extensions" Version="1.2.1" />
-    <PackageReference Include="CasCap.Common.Net" Version="1.2.1" />
-    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
+    <PackageReference Include="AzurePipelinesToGitHubActionsConverter.Core" Version="1.0.9" />
+    <PackageReference Include="CasCap.Common.Extensions" Version="1.2.7" />
+    <PackageReference Include="CasCap.Common.Net" Version="1.2.7" />
     <PackageReference Include="Microsoft.TeamFoundation.DistributedTask.WebApi" Version="16.170.0" />
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.170.0" />
     <PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.170.0" />
     <PackageReference Include="Microsoft.VisualStudio.Services.Release.Client" Version="16.170.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="semver" Version="2.1.0" />
-    <PackageReference Include="YamlDotNet" Version="11.2.1" />
+    <PackageReference Include="semver" Version="2.2.0" />
+    <PackageReference Include="YamlDotNet" Version="12.0.2" />
   </ItemGroup>
 
 </Project>
diff --git a/src/CasCap.Apis.AzureDevOps/Utilities/YamlPipelineGenerator.cs b/src/CasCap.Apis.AzureDevOps/Utilities/YamlPipelineGenerator.cs
@@ -20,6 +20,7 @@ public class YamlPipelineGenerator
     ConcurrentDictionary<TaskGroupVersion, Template> _taskGroupTemplateMap;//this collection is appended-to as the app iterates over the definitions
     readonly Dictionary<int, Microsoft.TeamFoundation.DistributedTask.WebApi.VariableGroup> _variableGroupMap;
     readonly bool _inlineTaskGroups;
+    readonly DeployPhaseTypes _phaseType;
 
     readonly string _templatesFolder = "AzureDevOpsTaskGroups";
 
@@ -36,7 +37,8 @@ enum VariableType
         Dictionary<TaskGroupVersion, TaskGroup> taskGroupMap,
         ConcurrentDictionary<TaskGroupVersion, Template> taskGroupTemplateMap,
         Dictionary<int, Microsoft.TeamFoundation.DistributedTask.WebApi.VariableGroup> variableGroupMap,
-        bool inlineTaskGroups
+        bool inlineTaskGroups,
+        DeployPhaseTypes phaseType
         )
     {
         _build = build;
@@ -46,6 +48,7 @@ bool inlineTaskGroups
         _taskGroupTemplateMap = taskGroupTemplateMap;
         _variableGroupMap = variableGroupMap;
         _inlineTaskGroups = inlineTaskGroups;
+        _phaseType = phaseType;
     }
 
     public Pipeline GenPipeline()
@@ -232,7 +235,7 @@ List<Job> GenJobs(ReleaseDefinitionEnvironment environment)
             var jobs = new List<Job>();
             var duplicatePhaseNames = environment.DeployPhases.Select(p => p.Name).Distinct().Count() > 1;
             var j = 0;
-            foreach (var phase in environment.DeployPhases.Where(p => p.PhaseType == DeployPhaseTypes.AgentBasedDeployment).OrderBy(p => p.Rank))
+            foreach (var phase in environment.DeployPhases.Where(p => p.PhaseType == _phaseType).OrderBy(p => p.Rank))
             {
                 var steps = new List<Step>(phase.WorkflowTasks.Count);
                 foreach (var task in phase.WorkflowTasks)
@@ -270,7 +273,7 @@ List<Step> GenSteps(TaskGroupStep task, Dictionary<string, string> parameters)
     List<Step> GenSteps(Guid Id, string displayName, string semver, IDictionary<string, string> inputs, IDictionary<string, string> env,
         string condition, bool continueOnError, int timeoutInMinutes, Dictionary<string, string> parameters = null)
     {
-        var version = SemVersion.Parse(semver.Replace(".*", ".0")).Major;
+        var version = SemVersion.Parse(semver.Replace(".*", ".0"), SemVersionStyles.OptionalPatch).Major;
         if (_taskMap.TryGetValue(Id, out var taskObjs) && taskObjs.TryGetValue(version, out var taskObj))
             return new List<Step>
             {

diff --git a/src/CasCap.DevOpsYamlizrCli/CasCap.DevOpsYamlizrCli.csproj b/src/CasCap.DevOpsYamlizrCli/CasCap.DevOpsYamlizrCli.csproj
@@ -15,15 +15,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="CasCap.Common.Threading" Version="1.2.1" />
+    <PackageReference Include="CasCap.Common.Threading" Version="1.2.7" />
     <PackageReference Include="ConsoleTables" Version="2.4.2" />
     <PackageReference Include="Figgle" Version="0.4.0" />
     <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.0.1" />
     <PackageReference Include="McMaster.Extensions.Hosting.CommandLine" Version="4.0.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
-    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
-    <PackageReference Include="ShellProgressBar" Version="5.1.0" />
+    <PackageReference Include="ShellProgressBar" Version="5.2.0" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/src/CasCap.DevOpsYamlizrCli/Commands/GenerateCommand.cs b/src/CasCap.DevOpsYamlizrCli/Commands/GenerateCommand.cs
@@ -37,6 +37,9 @@ public GenerateCommand(ILogger<GenerateCommand> logger, ILoggerFactory loggerFac
     [Option("--filter", Description = "Build/Release definition wildcard filter.")]
     public string filter { get; }
 
+    [Option("--phasetype", Description = "Filter deployment phases [default: AgentBasedDeployment]")]
+    public DeployPhaseTypes phaseType { get; set; } = DeployPhaseTypes.AgentBasedDeployment;
+
     [Option("--parallelism", Description = "Parallel execution mode (work in progress) [default: false]")]
     public bool parallelism { get; }
 
@@ -169,7 +172,15 @@ public async Task<int> OnExecuteAsync()
 
             var processedDefinitionCount = 0;
             if (parallelism || 1 == 1)//always parallel as this is pretty solid...
-                await buildDefinitionReferences.ForEachAsyncSemaphore(definitionReference => ProcessDefinition(definitionReference), Environment.ProcessorCount);
+#if NET6_0_OR_GREATER
+                await Parallel.ForEachAsync(buildDefinitionReferences, async (definitionReference, token) =>
+                {
+                    if (token.IsCancellationRequested) return;
+                    await ProcessDefinition(definitionReference);
+                });
+#else
+                await buildDefinitionReferences.ForEachAsyncSemaphore(definitionReference => ProcessDefinition(definitionReference));
+#endif
             else
                 foreach (var definitionReference in buildDefinitionReferences)
                     await ProcessDefinition(definitionReference);
@@ -223,7 +234,8 @@ void ProcessDefinition(BuildDefinition buildDefinition)
                     taskGroupMap,
                     taskGroupTemplateMap,
                     variableGroupMap,
-                    inlineTaskGroups
+                    inlineTaskGroups,
+                    phaseType
                     );
 
                 var pipeline = generator.GenPipeline();
@@ -254,7 +266,15 @@ void ProcessDefinition(BuildDefinition buildDefinition)
 
             var processedDefinitionCount = 0;
             if (parallelism)
-                await releaseDefinitions.ForEachAsyncSemaphore(releaseDefinition => ProcessDefinition(releaseDefinition), Environment.ProcessorCount);
+#if NET6_0_OR_GREATER
+                await Parallel.ForEachAsync(releaseDefinitions, async (releaseDefinition, token) =>
+                {
+                    if (token.IsCancellationRequested) return;
+                    await ProcessDefinition(releaseDefinition);
+                });
+#else
+                await releaseDefinitions.ForEachAsyncSemaphore(releaseDefinition => ProcessDefinition(releaseDefinition));
+#endif
             else
                 foreach (var releaseDefinition in releaseDefinitions)
                     await ProcessDefinition(releaseDefinition);
@@ -273,12 +293,13 @@ async Task ProcessDefinition(ReleaseDefinition releaseDefinition)
                     taskGroupMap,
                     taskGroupTemplateMap,
                     variableGroupMap,
-                    inlineTaskGroups
+                    inlineTaskGroups,
+                    phaseType
                     );
 
                 var pipeline = generator.GenPipeline();
                 if (pipeline is null)
-                    errors.Add($"Processing release definition id {releaseDefinition.Id} '{releaseDefinition.Name}' failed");
+                    errors.Add($"Processing release definition id {releaseDefinition.Id} '{releaseDefinition.Name}' failed (generated pipeline is null)");
                 else
                     results.Add((null, releaseDefinition, pipeline));
 
@@ -313,7 +334,15 @@ async Task ProcessDefinition(ReleaseDefinition releaseDefinition)
 
             var processedDefinitionCount = 0;
             if (parallelism || 1 == 1)//always parallel as this is pretty solid...
-                await definitions.ForEachAsyncSemaphore(result => ProcessDefinition(result), Environment.ProcessorCount);
+#if NET6_0_OR_GREATER
+                await Parallel.ForEachAsync(definitions, async (result, token) =>
+                {
+                    if (token.IsCancellationRequested) return;
+                    await ProcessDefinition(result);
+                });
+#else
+                await definitions.ForEachAsyncSemaphore(result => ProcessDefinition(result));
+#endif
             else
                 foreach (var result in definitions)
                     await ProcessDefinition(result);
@@ -348,7 +377,15 @@ async Task ProcessDefinition((BuildDefinition buildDefinition, ReleaseDefinition
 
             var processedDefinitionCount = 0;
             if (parallelism || 1 == 1)//always parallel as this is pretty solid...
-                await definitions.ForEachAsyncSemaphore(result => ProcessDefinition(result), Environment.ProcessorCount);
+#if NET6_0_OR_GREATER
+                await Parallel.ForEachAsync(definitions, async (result, token) =>
+                {
+                    if (token.IsCancellationRequested) return;
+                    await ProcessDefinition(result);
+                });
+#else
+                await definitions.ForEachAsyncSemaphore(result => ProcessDefinition(result));
+#endif
             else
                 foreach (var result in definitions)
                     await ProcessDefinition(result);

diff --git a/src/CasCap.DevOpsYamlizrCli/Properties/launchSettings.json b/src/CasCap.DevOpsYamlizrCli/Properties/launchSettings.json
@@ -1,7 +1,8 @@
 {
   "profiles": {
     "yamlizr": {
-      "commandName": "Project"
+      "commandName": "Project",
+      //"commandLineArgs": "generate -pat <your PAT here> -org <your AzDO organisation> -proj <your AzDO project> -out c:/temp/myoutputfolder",
     }
   }
 }