From 2bcf45182ed3334822a2907773858cf9b9e88d41 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Tue, 2 May 2023 15:47:14 -0700 Subject: [PATCH 01/31] Download manifest tool --- azure-pipelines.yml | 25 ++++++++++++++- build.ps1 | 64 +++++++++++++++++++++++++++++++++++--- test/E2E/Start-E2ETest.ps1 | 23 ++++++++------ 3 files changed, 97 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2d3664a..987655d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,30 @@ pool: - ImageOverride -equals $(imageName) steps: -- pwsh: ./test/E2E/Start-E2ETest.ps1 -UseCoreToolsBuildFromIntegrationTests +- pwsh: | + $simulateReleaseBuild = $null + if (-not([bool]::TryParse($env:SimulateReleaseBuild, [ref] $simulateReleaseBuild))) + { + throw "SimulateReleaseBuild can only be set to true or false." + } + + $isReleaseBuild = $false + if ($env:BuildSourceBranchName -like "release_*" -or $simulateReleaseBuild) + { + $isReleaseBuild = $true + } + Write-Host "Setting IsReleaseBuild to $isReleaseBuild because SimulateReleaseBuild is $env:SimulateReleaseBuild" + Write-Host "##vso[task.setvariable variable=IsReleaseBuild]$isReleaseBuild" + Write-Host "IsReleaseBuild: $isReleaseBuild" + +- pwsh: | + # We only generate an SBOM for release or simulated release builds + ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM + displayName: 'Build Durable SDK' + env: + SBOMUtilSASUrl: $(SBOMUtilSASUrl) + +- pwsh: ./test/E2E/Start-E2ETest.ps1 -NoBuild -UseCoreToolsBuildFromIntegrationTests env: AzureWebJobsStorage: $(AzureWebJobsStorage) displayName: 'Run E2E tests' diff --git a/build.ps1 b/build.ps1 index 71dc996..d82a434 100644 --- a/build.ps1 +++ b/build.ps1 @@ -3,7 +3,9 @@ param( [ValidateSet('Debug', 'Release')] [string] - $Configuration = 'Debug' + $Configuration = 'Debug', + [switch] + $AddSBOM ) $packageName = "AzureFunctions.PowerShell.Durable.SDK" @@ -24,6 +26,7 @@ $sharedDependenciesPath = "$outputPath/Dependencies/" $netCoreTFM = 'net6.0' $publishPathSuffix = "bin/$Configuration/$netCoreTFM/publish" +#region HELPER FUNCTIONS ========================================================================== function Write-Log { param ( @@ -53,6 +56,38 @@ function Write-Log Write-Host -ForegroundColor $foregroundColor $Message } +function Install-SBOMUtil +{ + if ([string]::IsNullOrEmpty($env:SBOMUtilSASUrl)) + { + throw "The `$SBOMUtilSASUrl environment variable cannot be null or empty when specifying the `$AddSBOM switch" + } + + $MANIFESTOOLNAME = "ManifestTool" + Write-Log "Installing $MANIFESTOOLNAME..." + + $MANIFESTOOL_DIRECTORY = Join-Path $PSScriptRoot $MANIFESTOOLNAME + Remove-Item -Recurse -Force $MANIFESTOOL_DIRECTORY -ErrorAction Ignore + + Invoke-RestMethod -Uri $env:SBOMUtilSASUrl -OutFile "$MANIFESTOOL_DIRECTORY.zip" + Expand-Archive "$MANIFESTOOL_DIRECTORY.zip" -DestinationPath $MANIFESTOOL_DIRECTORY + + $dllName = "Microsoft.ManifestTool.dll" + $manifestToolPath = Join-Path "$MANIFESTOOL_DIRECTORY" "$dllName" + + if (-not (Test-Path $manifestToolPath)) + { + throw "$MANIFESTOOL_DIRECTORY does not contain '$dllName'" + } + + Write-Log 'Done.' + + return $manifestToolPath +} + +#endregion + +#region BUILD ARTIFACTS =========================================================================== Write-Log "Build started..." Write-Log "Configuration: '$Configuration'`nOutput folder '$outputPath'`nShared dependencies folder: '$sharedDependenciesPath'" "Gray" @@ -89,13 +124,13 @@ foreach ($project in $projects.GetEnumerator()) { $commonFiles = [System.Collections.Generic.HashSet[string]]::new() Write-Log "Copying assemblies from the Durable Engine project into $sharedDependenciesPath" "Gray" -Get-ChildItem -Path "$durableEnginePath/$publishPathSuffix" | +Get-ChildItem -Path (Join-Path "$durableEnginePath" "$publishPathSuffix") | Where-Object { $_.Extension -in '.dll','.pdb' } | ForEach-Object { [void]$commonFiles.Add($_.Name); Copy-Item -LiteralPath $_.FullName -Destination $sharedDependenciesPath } # Copy all *unique* assemblies from Durable SDK into output directory Write-Log "Copying unique assemblies from the Durable SDK project into $outputPath" "Gray" -Get-ChildItem -Path "$shimPath/$publishPathSuffix" | +Get-ChildItem -Path (Join-Path "$shimPath" "$publishPathSuffix") | Where-Object { $_.Extension -in '.dll','.pdb' -and -not $commonFiles.Contains($_.Name) } | ForEach-Object { Copy-Item -LiteralPath $_.FullName -Destination $outputPath } @@ -103,4 +138,25 @@ Get-ChildItem -Path "$shimPath/$publishPathSuffix" | Write-Log "Copying PowerShell module and manifest from the Durable SDK source code into $outputPath" "Gray" Copy-Item -Path $powerShellModulePath -Destination $outputPath Copy-Item -Path $manifestPath -Destination $outputPath -Write-Log "Build succeeded!" \ No newline at end of file +Write-Log "Build succeeded!" +#endregion + +#region ADD SBOM ================================================================================== +if ($AddSBOM) { + # Install manifest tool + $manifestToolPath = Install-SBOMUtil + Write-Log "Manifest tool path": $manifestToolPath + + # Generate manifest + $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json") + $packageName = "AzureFunctions.PowerShell.Durable.SDK.nuspec" + + Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath" + & { dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName } + + # Discard telemetry generated + Remove-Item -Path $telemetryFilePath +} + +dotnet pack -c $Configuration +#endregion \ No newline at end of file diff --git a/test/E2E/Start-E2ETest.ps1 b/test/E2E/Start-E2ETest.ps1 index 7136b3a..c7ba685 100644 --- a/test/E2E/Start-E2ETest.ps1 +++ b/test/E2E/Start-E2ETest.ps1 @@ -4,6 +4,8 @@ # param ( + [Switch] + $NoBuild, [Switch] $UseCoreToolsBuildFromIntegrationTests, [Switch] @@ -148,16 +150,17 @@ if (-not $SkipCoreToolsDownload.IsPresent) $env:FUNC_PATH = $funcPath Write-Host "Set FUNC_PATH environment variable to $env:FUNC_PATH" -# For both integration build test runs and regular test runs, we copy binaries to durableApp/Modules -Write-Host "Building the DurableSDK module and copying binaries to the durableApp/Modules directory..." -$configuration = if ($env:CONFIGURATION) { $env:CONFIGURATION } else { 'Debug' } - -Push-Location "$PSScriptRoot/../.." -try { - & ./build.ps1 -Configuration 'Debug' -} -finally { - Pop-Location +if (-not $NoBuild.IsPresent) { + # For both integration build test runs and regular test runs, we copy binaries to durableApp/Modules + Write-Host "Building the DurableSDK module and copying binaries to the durableApp/Modules directory..." + + Push-Location "$PSScriptRoot/../.." + try { + & ./build.ps1 -Configuration 'Debug' + } + finally { + Pop-Location + } } Write-Host "Starting Core Tools..." From faa20596a4f49623e57f84a990880b42f761d309 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Tue, 2 May 2023 16:24:18 -0700 Subject: [PATCH 02/31] Remove packing step at the end of the build script --- azure-pipelines.yml | 2 ++ build.ps1 | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 987655d..1743dda 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -34,9 +34,11 @@ steps: Write-Host "Setting IsReleaseBuild to $isReleaseBuild because SimulateReleaseBuild is $env:SimulateReleaseBuild" Write-Host "##vso[task.setvariable variable=IsReleaseBuild]$isReleaseBuild" Write-Host "IsReleaseBuild: $isReleaseBuild" + displayName: Set IsReleaseBuild pipeline variable - pwsh: | # We only generate an SBOM for release or simulated release builds + Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM" ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM displayName: 'Build Durable SDK' env: diff --git a/build.ps1 b/build.ps1 index d82a434..1863a5c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -157,6 +157,4 @@ if ($AddSBOM) { # Discard telemetry generated Remove-Item -Path $telemetryFilePath } - -dotnet pack -c $Configuration #endregion \ No newline at end of file From 18cacf60afa16477cc837421218aa8788c3f229e Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 14:01:16 -0700 Subject: [PATCH 03/31] Add artifacts to publish --- azure-pipelines.yml | 15 +++++++++++++++ build.ps1 | 4 ++-- src/DurableSDK/DurableSDK.csproj | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1743dda..b1b7bb6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,12 @@ pool: demands: - ImageOverride -equals $(imageName) +variables: + artifactName: 'azure-functions-durable-ps' + # Every build will increment + buildNumber: $[counter('build', 001) ] + modulePath: './test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' + steps: - pwsh: | $simulateReleaseBuild = $null @@ -49,6 +55,15 @@ steps: AzureWebJobsStorage: $(AzureWebJobsStorage) displayName: 'Run E2E tests' +- pwsh: | + tar -czvf $(modulePath)/$(artifactName)-$(buildNumber).tar.gz -C $(modulePath) + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: package' + inputs: + PathtoPublish: $(modulePath) + ArtifactName: $(artifactName) + - task: PublishTestResults@2 inputs: testResultsFormat: 'VSTest' diff --git a/build.ps1 b/build.ps1 index 1863a5c..1927d80 100644 --- a/build.ps1 +++ b/build.ps1 @@ -149,10 +149,10 @@ if ($AddSBOM) { # Generate manifest $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json") - $packageName = "AzureFunctions.PowerShell.Durable.SDK.nuspec" + $packageName = "AzureFunctions.PowerShell.Durable.SDK" Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath" - & { dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName } + dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName # Discard telemetry generated Remove-Item -Path $telemetryFilePath diff --git a/src/DurableSDK/DurableSDK.csproj b/src/DurableSDK/DurableSDK.csproj index 9e3b3b5..ed4cd9f 100644 --- a/src/DurableSDK/DurableSDK.csproj +++ b/src/DurableSDK/DurableSDK.csproj @@ -8,6 +8,6 @@ - + From e8fe839c12ae3befcea1be0653bbfc0d160d14be Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 14:22:03 -0700 Subject: [PATCH 04/31] Debug Linux --- azure-pipelines.yml | 6 ++++-- test/E2E/durableApp/SimpleOrchestrator/run.ps1 | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b1b7bb6..d6696c9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ steps: $simulateReleaseBuild = $null if (-not([bool]::TryParse($env:SimulateReleaseBuild, [ref] $simulateReleaseBuild))) { - throw "SimulateReleaseBuild can only be set to true or false." + throw "SimulateReleaseBuild set to $env:SimulateReleaseBuild. It can only be set to true or false." } $isReleaseBuild = $false @@ -57,9 +57,11 @@ steps: - pwsh: | tar -czvf $(modulePath)/$(artifactName)-$(buildNumber).tar.gz -C $(modulePath) + displayName: 'Tar build artifacts' + condition: eq(variables['IsReleaseBuild'], 'true') - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: package' + displayName: 'Publish .tar artifact' inputs: PathtoPublish: $(modulePath) ArtifactName: $(artifactName) diff --git a/test/E2E/durableApp/SimpleOrchestrator/run.ps1 b/test/E2E/durableApp/SimpleOrchestrator/run.ps1 index dca78f0..19c4ae4 100644 --- a/test/E2E/durableApp/SimpleOrchestrator/run.ps1 +++ b/test/E2E/durableApp/SimpleOrchestrator/run.ps1 @@ -6,4 +6,4 @@ $ErrorActionPreference = 'Stop' $output = Invoke-DurableActivity -FunctionName "Hello" -Input "Tokyo" -return $output +$output From 04d7138b717a626f397bec6e8ac65d59e9badbdf Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 14:28:31 -0700 Subject: [PATCH 05/31] Temporarily disable suborchestrator E2E test --- azure-pipelines.yml | 2 +- .../DurableClientTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d6696c9..cacbe27 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ steps: $simulateReleaseBuild = $null if (-not([bool]::TryParse($env:SimulateReleaseBuild, [ref] $simulateReleaseBuild))) { - throw "SimulateReleaseBuild set to $env:SimulateReleaseBuild. It can only be set to true or false." + throw "SimulateReleaseBuild set to $($env:SimulateReleaseBuild). It can only be set to true or false." } $isReleaseBuild = $false diff --git a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs index c18f9d1..a221c60 100644 --- a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs +++ b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs @@ -51,7 +51,7 @@ await ValidateDurableWorkflowResults( }); } - [Fact] + [SkippableFact] public async Task DurableSubOrchestratorCompletes() { var initialResponse = await Utilities.GetHttpStartResponse("SubOrchestrator"); From aeac99a54bff5277c9ddf75cd20290be64f79fb2 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 14:33:47 -0700 Subject: [PATCH 06/31] Log SimulateReleaseBuild value --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cacbe27..5b34aa0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,9 +27,10 @@ variables: steps: - pwsh: | $simulateReleaseBuild = $null + Write-Host "SimulateReleaseBuild set to $env:SimulateReleaseBuild" if (-not([bool]::TryParse($env:SimulateReleaseBuild, [ref] $simulateReleaseBuild))) { - throw "SimulateReleaseBuild set to $($env:SimulateReleaseBuild). It can only be set to true or false." + throw "SimulateReleaseBuild can only be set to true or false." } $isReleaseBuild = $false From 1a76b35e450672128ac97bd3a13c9f43e2e9230b Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 14:44:55 -0700 Subject: [PATCH 07/31] Pass SimulateReleaseBuild variable to Linux pipeline --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5b34aa0..b0c094e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,6 +42,8 @@ steps: Write-Host "##vso[task.setvariable variable=IsReleaseBuild]$isReleaseBuild" Write-Host "IsReleaseBuild: $isReleaseBuild" displayName: Set IsReleaseBuild pipeline variable + env: + SimulateReleaseBuild: $(SimulateReleaseBuild) - pwsh: | # We only generate an SBOM for release or simulated release builds From 7ea6706a348e311083bf8a296aae7e2652803418 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 14:53:27 -0700 Subject: [PATCH 08/31] Remove test and update pipeline conditions --- azure-pipelines.yml | 3 +- .../DurableClientTests.cs | 66 +++++++++---------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b0c094e..ab7ae1e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -61,13 +61,14 @@ steps: - pwsh: | tar -czvf $(modulePath)/$(artifactName)-$(buildNumber).tar.gz -C $(modulePath) displayName: 'Tar build artifacts' - condition: eq(variables['IsReleaseBuild'], 'true') + condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' inputs: PathtoPublish: $(modulePath) ArtifactName: $(artifactName) + condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - task: PublishTestResults@2 inputs: diff --git a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs index a221c60..2d2cd95 100644 --- a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs +++ b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs @@ -51,41 +51,41 @@ await ValidateDurableWorkflowResults( }); } - [SkippableFact] - public async Task DurableSubOrchestratorCompletes() - { - var initialResponse = await Utilities.GetHttpStartResponse("SubOrchestrator"); - Assert.Equal(HttpStatusCode.Accepted, initialResponse.StatusCode); + // [Fact] + // public async Task DurableSubOrchestratorCompletes() + // { + // var initialResponse = await Utilities.GetHttpStartResponse("SubOrchestrator"); + // Assert.Equal(HttpStatusCode.Accepted, initialResponse.StatusCode); - var location = initialResponse.Headers.Location; - Assert.NotNull(location); + // var location = initialResponse.Headers.Location; + // Assert.NotNull(location); - await ValidateDurableWorkflowResults( - initialResponse, - validateInitialResponse: (dynamic initialStatusResponseBody) => - { - Assert.NotNull(initialStatusResponseBody.id); - var statusQueryGetUri = (string)initialStatusResponseBody.statusQueryGetUri; - Assert.Equal(location?.ToString(), statusQueryGetUri); - Assert.NotNull(initialStatusResponseBody.sendEventPostUri); - Assert.NotNull(initialStatusResponseBody.purgeHistoryDeleteUri); - Assert.NotNull(initialStatusResponseBody.terminatePostUri); - Assert.NotNull(initialStatusResponseBody.rewindPostUri); - }, - validateIntermediateResponse: (dynamic intermediateStatusResponseBody) => - { - var runtimeStatus = (string)intermediateStatusResponseBody.runtimeStatus; - Assert.True( - runtimeStatus == "Running" || runtimeStatus == "Pending", - $"Unexpected runtime status: {runtimeStatus}"); - }, - validateFinalResponse: (dynamic finalStatusResponseBody) => - { - Assert.Equal("Completed", (string)finalStatusResponseBody.runtimeStatus); - Assert.Equal("Hello Tokyo", finalStatusResponseBody.output[0].ToString()); - Assert.Equal("Hello Seattle", finalStatusResponseBody.output[1].ToString()); - }); - } + // await ValidateDurableWorkflowResults( + // initialResponse, + // validateInitialResponse: (dynamic initialStatusResponseBody) => + // { + // Assert.NotNull(initialStatusResponseBody.id); + // var statusQueryGetUri = (string)initialStatusResponseBody.statusQueryGetUri; + // Assert.Equal(location?.ToString(), statusQueryGetUri); + // Assert.NotNull(initialStatusResponseBody.sendEventPostUri); + // Assert.NotNull(initialStatusResponseBody.purgeHistoryDeleteUri); + // Assert.NotNull(initialStatusResponseBody.terminatePostUri); + // Assert.NotNull(initialStatusResponseBody.rewindPostUri); + // }, + // validateIntermediateResponse: (dynamic intermediateStatusResponseBody) => + // { + // var runtimeStatus = (string)intermediateStatusResponseBody.runtimeStatus; + // Assert.True( + // runtimeStatus == "Running" || runtimeStatus == "Pending", + // $"Unexpected runtime status: {runtimeStatus}"); + // }, + // validateFinalResponse: (dynamic finalStatusResponseBody) => + // { + // Assert.Equal("Completed", (string)finalStatusResponseBody.runtimeStatus); + // Assert.Equal("Hello Tokyo", finalStatusResponseBody.output[0].ToString()); + // Assert.Equal("Hello Seattle", finalStatusResponseBody.output[1].ToString()); + // }); + // } [Fact] public async Task DurableClientTerminatesOrchestration() From a709130e7d3bcdb7db5e483cff6bc7ebc4769bf2 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 15:02:11 -0700 Subject: [PATCH 09/31] Alter tar command --- azure-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ab7ae1e..e8156ba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,7 +59,11 @@ steps: displayName: 'Run E2E tests' - pwsh: | - tar -czvf $(modulePath)/$(artifactName)-$(buildNumber).tar.gz -C $(modulePath) + $modulePath = $env:modulePath + $artifactName = $env:artifactName + $buildNumber = $env:buildNumber + Write-Host "Running 'tar -czvf $modulePath/$artifactName-$buildNumber).tar.gz -C $modulePath/**'" + tar -czvf $modulePath/$artifactName-$buildNumber).tar.gz -C $modulePath/** displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) From bb25ffaf3f532135e94eb9b99ade248a764ef1fa Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 15:08:41 -0700 Subject: [PATCH 10/31] Correct typo --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e8156ba..b842967 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -62,8 +62,8 @@ steps: $modulePath = $env:modulePath $artifactName = $env:artifactName $buildNumber = $env:buildNumber - Write-Host "Running 'tar -czvf $modulePath/$artifactName-$buildNumber).tar.gz -C $modulePath/**'" - tar -czvf $modulePath/$artifactName-$buildNumber).tar.gz -C $modulePath/** + Write-Host "Running 'tar -czvf $modulePath/$artifactName-$buildNumber.tar.gz -C $modulePath/**'" + tar -czvf $modulePath/$artifactName-$buildNumber.tar.gz -C $modulePath/** displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) From 64939b770aec3550c71adf3c8d30b8779d952cb2 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 15:36:32 -0700 Subject: [PATCH 11/31] Add package folder --- .gitignore | 3 +++ azure-pipelines.yml | 13 ++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 3cf8180..05418c1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,8 @@ local.settings.json # Temporary output folder for Durable SDK binaries **/src/out +# Package folder for .tar file +/package/** + # VS publish settings *.pubxml \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b842967..3c919d3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,7 +47,7 @@ steps: - pwsh: | # We only generate an SBOM for release or simulated release builds - Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM" + Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM..." ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM displayName: 'Build Durable SDK' env: @@ -62,16 +62,19 @@ steps: $modulePath = $env:modulePath $artifactName = $env:artifactName $buildNumber = $env:buildNumber - Write-Host "Running 'tar -czvf $modulePath/$artifactName-$buildNumber.tar.gz -C $modulePath/**'" - tar -czvf $modulePath/$artifactName-$buildNumber.tar.gz -C $modulePath/** + + Write-Host "Creating 'package' directory..." + New-Item "./package" -ItemType Directory + Write-Host "Running 'tar -czvf ./package/$artifactName-$buildNumber.tar.gz -C $modulePath .'..." + tar -czvf ./package/$artifactName-$buildNumber.tar.gz -C $modulePath . displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' inputs: - PathtoPublish: $(modulePath) - ArtifactName: $(artifactName) + PathtoPublish: ./package + ArtifactName: $(artifactName)-$(buildNumber).tar.gz condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - task: PublishTestResults@2 From b563d4ea6258fcef5f089d0fd6755518b6fd141b Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 15:55:55 -0700 Subject: [PATCH 12/31] Add sources directory and environment variables for Linux --- azure-pipelines.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3c919d3..0e89b35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,21 +59,27 @@ steps: displayName: 'Run E2E tests' - pwsh: | - $modulePath = $env:modulePath $artifactName = $env:artifactName $buildNumber = $env:buildNumber + $modulePath = $env:modulePath + $src = $env:src Write-Host "Creating 'package' directory..." - New-Item "./package" -ItemType Directory - Write-Host "Running 'tar -czvf ./package/$artifactName-$buildNumber.tar.gz -C $modulePath .'..." - tar -czvf ./package/$artifactName-$buildNumber.tar.gz -C $modulePath . + New-Item "$src/package" -ItemType Directory + Write-Host "Running 'tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath .'..." + tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath . displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) + env: + artifactName: $(artifactName) + buildNumber: $(buildNumber) + modulePath: $(modulePath) + src: $(Build.SourcesDirectory) - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' inputs: - PathtoPublish: ./package + PathtoPublish: $(Build.SourcesDirectory)/package ArtifactName: $(artifactName)-$(buildNumber).tar.gz condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) From b5abd7f9c72334b2962f5056b6583e0c3d54687f Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 16:02:46 -0700 Subject: [PATCH 13/31] Correct path typo --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0e89b35..141c1d9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ variables: artifactName: 'azure-functions-durable-ps' # Every build will increment buildNumber: $[counter('build', 001) ] - modulePath: './test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' + modulePath: './test/E2E/durableApp\Modules/AzureFunctions.PowerShell.Durable.SDK' steps: - pwsh: | From cb595eb1608790a4c9ae2bf2c35e56e3aa78aee7 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 16:09:17 -0700 Subject: [PATCH 14/31] Correct another path typo --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 141c1d9..0e89b35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -22,7 +22,7 @@ variables: artifactName: 'azure-functions-durable-ps' # Every build will increment buildNumber: $[counter('build', 001) ] - modulePath: './test/E2E/durableApp\Modules/AzureFunctions.PowerShell.Durable.SDK' + modulePath: './test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' steps: - pwsh: | From 21ee9fda3505fe66350d3d2a0947eafa7ffc00d9 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 16:38:46 -0700 Subject: [PATCH 15/31] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0e89b35..700401c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,23 +58,15 @@ steps: AzureWebJobsStorage: $(AzureWebJobsStorage) displayName: 'Run E2E tests' -- pwsh: | - $artifactName = $env:artifactName - $buildNumber = $env:buildNumber - $modulePath = $env:modulePath - $src = $env:src - - Write-Host "Creating 'package' directory..." - New-Item "$src/package" -ItemType Directory - Write-Host "Running 'tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath .'..." - tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath . +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(Build.SourcesDirectory)\test\E2E\durableApp\Modules\AzureFunctions.PowerShell.Durable.SDK' + includeRootFolder: false + archiveType: 'tar' + archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName)-$(buildNumber).tar.gz' + replaceExistingArchive: true displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - env: - artifactName: $(artifactName) - buildNumber: $(buildNumber) - modulePath: $(modulePath) - src: $(Build.SourcesDirectory) - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' From d565beff6abd8ad8a80fc2d06a3f779d91e7a3b8 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 16:57:38 -0700 Subject: [PATCH 16/31] Trying different path specifications --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 700401c..259e89e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,10 +60,10 @@ steps: - task: ArchiveFiles@2 inputs: - rootFolderOrFile: '$(Build.SourcesDirectory)\test\E2E\durableApp\Modules\AzureFunctions.PowerShell.Durable.SDK' + rootFolderOrFile: 'test\E2E\durableApp\Modules\AzureFunctions.PowerShell.Durable.SDK' includeRootFolder: false archiveType: 'tar' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName)-$(buildNumber).tar.gz' + archiveFile: '$(Build.ArtifactStagingDirectory)\$(artifactName)-$(buildNumber).tar.gz' replaceExistingArchive: true displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) @@ -71,7 +71,7 @@ steps: - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' inputs: - PathtoPublish: $(Build.SourcesDirectory)/package + PathtoPublish: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifactName)-$(buildNumber).tar.gz condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) From 7edc03ec8818d2f030a99feb948f9d43107f74a5 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Thu, 4 May 2023 17:04:20 -0700 Subject: [PATCH 17/31] Debugging --- azure-pipelines.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 259e89e..81817a4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,20 +58,30 @@ steps: AzureWebJobsStorage: $(AzureWebJobsStorage) displayName: 'Run E2E tests' -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: 'test\E2E\durableApp\Modules\AzureFunctions.PowerShell.Durable.SDK' - includeRootFolder: false - archiveType: 'tar' - archiveFile: '$(Build.ArtifactStagingDirectory)\$(artifactName)-$(buildNumber).tar.gz' - replaceExistingArchive: true +- pwsh: | + $artifactName = $env:artifactName + $buildNumber = $env:buildNumber + $modulePath = $env:modulePath + $src = $env:src + + Write-Host "Creating '$src/package' directory..." + New-Item "$src/package" -ItemType Directory + cd ./test/E2E/durableApp + ls + Write-Host "Running 'tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath .'..." + tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath . displayName: 'Tar build artifacts' condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) + env: + artifactName: $(artifactName) + buildNumber: $(buildNumber) + modulePath: $(modulePath) + src: $(Build.SourcesDirectory) - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory) + PathtoPublish: $(Build.SourcesDirectory)/package ArtifactName: $(artifactName)-$(buildNumber).tar.gz condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) From 7cc4519e2797e233b389ca74d788b5178a45ff97 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 10:45:11 -0700 Subject: [PATCH 18/31] Add release builds to Module subfolder --- azure-pipelines.yml | 29 +++++++++-------------------- build.ps1 | 9 +++------ test/E2E/Start-E2ETest.ps1 | 6 +++--- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 81817a4..e6d532a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,30 +53,19 @@ steps: env: SBOMUtilSASUrl: $(SBOMUtilSASUrl) -- pwsh: ./test/E2E/Start-E2ETest.ps1 -NoBuild -UseCoreToolsBuildFromIntegrationTests +- pwsh: | + ./test/E2E/Start-E2ETest.ps1 -NoBuild -UseCoreToolsBuildFromIntegrationTests env: AzureWebJobsStorage: $(AzureWebJobsStorage) displayName: 'Run E2E tests' -- pwsh: | - $artifactName = $env:artifactName - $buildNumber = $env:buildNumber - $modulePath = $env:modulePath - $src = $env:src - - Write-Host "Creating '$src/package' directory..." - New-Item "$src/package" -ItemType Directory - cd ./test/E2E/durableApp - ls - Write-Host "Running 'tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath .'..." - tar -czvf $src/package/$artifactName-$buildNumber.tar.gz -C $modulePath . - displayName: 'Tar build artifacts' - condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - env: - artifactName: $(artifactName) - buildNumber: $(buildNumber) - modulePath: $(modulePath) - src: $(Build.SourcesDirectory) +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(Build.SourcesDirectory)/test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' + includeRootFolder: false + archiveType: 'tar' + archiveFile: '$(Build.ArtifactStagingDirectory)/azure-functions-durable-powershell-$(Build.SourceVersion).tar.gz' + replaceExistingArchive: true - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' diff --git a/build.ps1 b/build.ps1 index 1927d80..883b340 100644 --- a/build.ps1 +++ b/build.ps1 @@ -15,12 +15,9 @@ $durableAppPath = "$PSScriptRoot/test/E2E/durableApp/Modules/$packageName" $powerShellModulePath = "$PSScriptRoot/src/$packageName.psm1" $manifestPath = "$PSScriptRoot/src/$packageName.psd1" -$outputPath = "$PSScriptRoot/src/out/" -if ($Configuration -eq "Debug") -{ - # Publish directly to the test durable app for testing - $outputPath = $durableAppPath -} +# Publish directly to the test durable app for testing +$outputPath = $durableAppPath + $sharedDependenciesPath = "$outputPath/Dependencies/" $netCoreTFM = 'net6.0' diff --git a/test/E2E/Start-E2ETest.ps1 b/test/E2E/Start-E2ETest.ps1 index c7ba685..3210538 100644 --- a/test/E2E/Start-E2ETest.ps1 +++ b/test/E2E/Start-E2ETest.ps1 @@ -100,11 +100,11 @@ Write-Host "Deleting $FUNC_CLI_DIRECTORY if it exists..." Remove-Item -Force "$FUNC_CLI_DIRECTORY.zip" -ErrorAction Ignore Remove-Item -Recurse -Force $FUNC_CLI_DIRECTORY -ErrorAction Ignore -if (-not $SkipCoreToolsDownload.IsPresent) +if (-not $SkipCoreToolsDownload) { Write-Host "Downloading Core Tools because SkipCoreToolsDownload switch parameter is not present..." $coreToolsDownloadURL = $null - if ($UseCoreToolsBuildFromIntegrationTests.IsPresent) + if ($UseCoreToolsBuildFromIntegrationTests) { $coreToolsDownloadURL = "https://functionsintegclibuilds.blob.core.windows.net/builds/$FUNC_RUNTIME_VERSION/latest/Azure.Functions.Cli.$os-$arch.zip" $env:CORE_TOOLS_URL = "https://functionsintegclibuilds.blob.core.windows.net/builds/$FUNC_RUNTIME_VERSION/latest" @@ -150,7 +150,7 @@ if (-not $SkipCoreToolsDownload.IsPresent) $env:FUNC_PATH = $funcPath Write-Host "Set FUNC_PATH environment variable to $env:FUNC_PATH" -if (-not $NoBuild.IsPresent) { +if (-not $NoBuild) { # For both integration build test runs and regular test runs, we copy binaries to durableApp/Modules Write-Host "Building the DurableSDK module and copying binaries to the durableApp/Modules directory..." From ceee24d15ba5b409ec0a95c27f5a250d71a07e9a Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 11:24:44 -0700 Subject: [PATCH 19/31] Correct artifacts directory --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e6d532a..2b9a227 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ pool: - ImageOverride -equals $(imageName) variables: - artifactName: 'azure-functions-durable-ps' + artifactName: 'azure-functions-durable-powershell-$(Build.SourceVersion)' # Every build will increment buildNumber: $[counter('build', 001) ] modulePath: './test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' @@ -64,14 +64,14 @@ steps: rootFolderOrFile: '$(Build.SourcesDirectory)/test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' includeRootFolder: false archiveType: 'tar' - archiveFile: '$(Build.ArtifactStagingDirectory)/azure-functions-durable-powershell-$(Build.SourceVersion).tar.gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName).tar.gz' replaceExistingArchive: true - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' inputs: - PathtoPublish: $(Build.SourcesDirectory)/package - ArtifactName: $(artifactName)-$(buildNumber).tar.gz + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifactName).tar.gz condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) - task: PublishTestResults@2 From 432eb0bc56e88ff85659f4229ebf4e75ae808867 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 13:02:31 -0700 Subject: [PATCH 20/31] Generate SBOM --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2b9a227..48b172b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,10 +47,11 @@ steps: - pwsh: | # We only generate an SBOM for release or simulated release builds - Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM..." - ./build.ps1 -Configuration Release -AddSBOM:$shouldAddSBOM + Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$env:IsReleaseBuild..." + ./build.ps1 -Configuration Release -AddSBOM:$env:IsReleaseBuild displayName: 'Build Durable SDK' env: + IsReleaseBuild: $(IsReleaseBuild) SBOMUtilSASUrl: $(SBOMUtilSASUrl) - pwsh: | From e4ded3cb1479498d06c94a96f0ac5902ed922f9b Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 13:12:52 -0700 Subject: [PATCH 21/31] Parse IsReleaseBuild as a bool --- azure-pipelines.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 48b172b..bbd6986 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -46,11 +46,18 @@ steps: SimulateReleaseBuild: $(SimulateReleaseBuild) - pwsh: | + Write-Host "IsReleaseBuild set to $env:IsReleaseBuild" + if (-not([bool]::TryParse($env:IsReleaseBuild, [ref] $isReleaseBuild))) + { + throw "SimulateReleaseBuild can only be set to true or false." + } # We only generate an SBOM for release or simulated release builds - Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$env:IsReleaseBuild..." - ./build.ps1 -Configuration Release -AddSBOM:$env:IsReleaseBuild + Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$isReleaseBuild..." + ./build.ps1 -Configuration Release -AddSBOM:$isReleaseBuild displayName: 'Build Durable SDK' env: + # We include IsReleaseBuild as an environment variable since Linux agents don't seem to support including + # pipeline variables in scripts with the $(variable) syntax IsReleaseBuild: $(IsReleaseBuild) SBOMUtilSASUrl: $(SBOMUtilSASUrl) From 240d2804c35b871221177dae4698310d893b0675 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 13:19:30 -0700 Subject: [PATCH 22/31] Initialize IsReleaseBuild --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bbd6986..5e31050 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,10 +47,12 @@ steps: - pwsh: | Write-Host "IsReleaseBuild set to $env:IsReleaseBuild" + $isReleaseBuild = $false if (-not([bool]::TryParse($env:IsReleaseBuild, [ref] $isReleaseBuild))) { throw "SimulateReleaseBuild can only be set to true or false." } + # We only generate an SBOM for release or simulated release builds Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$isReleaseBuild..." ./build.ps1 -Configuration Release -AddSBOM:$isReleaseBuild From 924ac3a26ec83a9d4ab491340594cb6a45c6257e Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:24:54 -0700 Subject: [PATCH 23/31] Correct typo in build script --- build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 883b340..e907d49 100644 --- a/build.ps1 +++ b/build.ps1 @@ -142,7 +142,7 @@ Write-Log "Build succeeded!" if ($AddSBOM) { # Install manifest tool $manifestToolPath = Install-SBOMUtil - Write-Log "Manifest tool path": $manifestToolPath + Write-Log "Manifest tool path: $manifestToolPath" # Generate manifest $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json") From 97088519139f24971959fafd0d261fa8537b033f Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:31:33 -0700 Subject: [PATCH 24/31] Remove telemetry file --- build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index e907d49..0cb1dd6 100644 --- a/build.ps1 +++ b/build.ps1 @@ -152,6 +152,6 @@ if ($AddSBOM) { dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName # Discard telemetry generated - Remove-Item -Path $telemetryFilePath + Remove-Item -Path (Join-Path $outputPath $telemetryFilePath) -ErrorAction Ignore } #endregion \ No newline at end of file From a26d4cbf4238439ffcce5246ea99139e0c32421d Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:33:41 -0700 Subject: [PATCH 25/31] Create telemetry file --- build.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.ps1 b/build.ps1 index 0cb1dd6..05abf45 100644 --- a/build.ps1 +++ b/build.ps1 @@ -146,12 +146,13 @@ if ($AddSBOM) { # Generate manifest $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json") + New-Item -ItemType File -Path $telemetryFilePath $packageName = "AzureFunctions.PowerShell.Durable.SDK" - Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath" + Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName" dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName # Discard telemetry generated - Remove-Item -Path (Join-Path $outputPath $telemetryFilePath) -ErrorAction Ignore + Remove-Item -Path $telemetryFilePath -ErrorAction Ignore } #endregion \ No newline at end of file From c9fefab29c597381ccf221fa3b6d7a4edfc9952a Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:36:53 -0700 Subject: [PATCH 26/31] Add manifestToolPath --- build.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.ps1 b/build.ps1 index 05abf45..42fc1e2 100644 --- a/build.ps1 +++ b/build.ps1 @@ -149,8 +149,8 @@ if ($AddSBOM) { New-Item -ItemType File -Path $telemetryFilePath $packageName = "AzureFunctions.PowerShell.Durable.SDK" - Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName" - dotnet $manifestTool generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName + Write-Log "Running: dotnet $manifestToolPath generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName" + dotnet $manifestToolPath generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName # Discard telemetry generated Remove-Item -Path $telemetryFilePath -ErrorAction Ignore From 11a16ac1f3468745f1e13467b6971c6eaffefeb5 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:37:27 -0700 Subject: [PATCH 27/31] Remove faulty file generation --- build.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/build.ps1 b/build.ps1 index 42fc1e2..45f880d 100644 --- a/build.ps1 +++ b/build.ps1 @@ -146,7 +146,6 @@ if ($AddSBOM) { # Generate manifest $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json") - New-Item -ItemType File -Path $telemetryFilePath $packageName = "AzureFunctions.PowerShell.Durable.SDK" Write-Log "Running: dotnet $manifestToolPath generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName" From d4fe3c8bca7ba748c2df42f5816dcb55623c3684 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:52:35 -0700 Subject: [PATCH 28/31] Install .NET 3.1 --- azure-pipelines.yml | 5 ++ build.ps1 | 63 +------------------- pipelineUtilities.psm1 | 128 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 61 deletions(-) create mode 100644 pipelineUtilities.psm1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5e31050..9dc3928 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -45,6 +45,11 @@ steps: env: SimulateReleaseBuild: $(SimulateReleaseBuild) +- pwsh: | + Import-Module ".\pipelineUtilities.psm1" -Force + Install-Dotnet + displayName: 'Install .NET 3.1' + - pwsh: | Write-Host "IsReleaseBuild set to $env:IsReleaseBuild" $isReleaseBuild = $false diff --git a/build.ps1 b/build.ps1 index 45f880d..74ae193 100644 --- a/build.ps1 +++ b/build.ps1 @@ -8,6 +8,8 @@ param( $AddSBOM ) +Import-Module "$PSScriptRoot\pipelineUtilities.psm1" -Force + $packageName = "AzureFunctions.PowerShell.Durable.SDK" $shimPath = "$PSScriptRoot/src/DurableSDK" $durableEnginePath = "$PSScriptRoot/src/DurableEngine" @@ -23,67 +25,6 @@ $sharedDependenciesPath = "$outputPath/Dependencies/" $netCoreTFM = 'net6.0' $publishPathSuffix = "bin/$Configuration/$netCoreTFM/publish" -#region HELPER FUNCTIONS ========================================================================== -function Write-Log -{ - param ( - [Parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - [System.String] - $Message, - - [Switch] - $Warning, - - [Switch] - $Throw, - - [System.String] - $Color - ) - - $Message = (Get-Date -Format G) + " -- $Message" - - if ($Throw) - { - throw $Message - } - - $foregroundColor = if ($Warning.IsPresent) { 'Yellow' } elseif ($Color) { $Color } else { 'Green' } - Write-Host -ForegroundColor $foregroundColor $Message -} - -function Install-SBOMUtil -{ - if ([string]::IsNullOrEmpty($env:SBOMUtilSASUrl)) - { - throw "The `$SBOMUtilSASUrl environment variable cannot be null or empty when specifying the `$AddSBOM switch" - } - - $MANIFESTOOLNAME = "ManifestTool" - Write-Log "Installing $MANIFESTOOLNAME..." - - $MANIFESTOOL_DIRECTORY = Join-Path $PSScriptRoot $MANIFESTOOLNAME - Remove-Item -Recurse -Force $MANIFESTOOL_DIRECTORY -ErrorAction Ignore - - Invoke-RestMethod -Uri $env:SBOMUtilSASUrl -OutFile "$MANIFESTOOL_DIRECTORY.zip" - Expand-Archive "$MANIFESTOOL_DIRECTORY.zip" -DestinationPath $MANIFESTOOL_DIRECTORY - - $dllName = "Microsoft.ManifestTool.dll" - $manifestToolPath = Join-Path "$MANIFESTOOL_DIRECTORY" "$dllName" - - if (-not (Test-Path $manifestToolPath)) - { - throw "$MANIFESTOOL_DIRECTORY does not contain '$dllName'" - } - - Write-Log 'Done.' - - return $manifestToolPath -} - -#endregion - #region BUILD ARTIFACTS =========================================================================== Write-Log "Build started..." Write-Log "Configuration: '$Configuration'`nOutput folder '$outputPath'`nShared dependencies folder: '$sharedDependenciesPath'" "Gray" diff --git a/pipelineUtilities.psm1 b/pipelineUtilities.psm1 new file mode 100644 index 0000000..f2332f9 --- /dev/null +++ b/pipelineUtilities.psm1 @@ -0,0 +1,128 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +using namespace System.Runtime.InteropServices + +$DotnetSDKVersionRequirements = @{ + + # .NET SDK 3.1 is required by the Microsoft.ManifestTool.dll tool + '3.1' = @{ + MinimalPatch = '415' + DefaultPatch = '415' + } +} + +function Write-Log +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [System.String] + $Message, + + [Switch] + $Warning, + + [Switch] + $Throw, + + [System.String] + $Color + ) + + $Message = (Get-Date -Format G) + " -- $Message" + + if ($Throw) + { + throw $Message + } + + $foregroundColor = if ($Warning.IsPresent) { 'Yellow' } elseif ($Color) { $Color } else { 'Green' } + Write-Host -ForegroundColor $foregroundColor $Message +} + +function Install-SBOMUtil +{ + if ([string]::IsNullOrEmpty($env:SBOMUtilSASUrl)) + { + throw "The `$SBOMUtilSASUrl environment variable cannot be null or empty when specifying the `$AddSBOM switch" + } + + $MANIFESTOOLNAME = "ManifestTool" + Write-Log "Installing $MANIFESTOOLNAME..." + + $MANIFESTOOL_DIRECTORY = Join-Path $PSScriptRoot $MANIFESTOOLNAME + Remove-Item -Recurse -Force $MANIFESTOOL_DIRECTORY -ErrorAction Ignore + + Invoke-RestMethod -Uri $env:SBOMUtilSASUrl -OutFile "$MANIFESTOOL_DIRECTORY.zip" + Expand-Archive "$MANIFESTOOL_DIRECTORY.zip" -DestinationPath $MANIFESTOOL_DIRECTORY + + $dllName = "Microsoft.ManifestTool.dll" + $manifestToolPath = Join-Path "$MANIFESTOOL_DIRECTORY" "$dllName" + + if (-not (Test-Path $manifestToolPath)) + { + throw "$MANIFESTOOL_DIRECTORY does not contain '$dllName'" + } + + Write-Log 'Done.' + + return $manifestToolPath +} + + +function AddLocalDotnetDirPath { + $LocalDotnetDirPath = if ($IsWindows) { "$env:ProgramFiles/dotnet" } else { "/usr/share/dotnet" } + if (($env:PATH -split [IO.Path]::PathSeparator) -notcontains $LocalDotnetDirPath) { + $env:PATH = $LocalDotnetDirPath + [IO.Path]::PathSeparator + $env:PATH + } +} + +function Find-Dotnet +{ + AddLocalDotnetDirPath + $listSdksOutput = dotnet --list-sdks + $installedDotnetSdks = $listSdksOutput | ForEach-Object { $_.Split(" ")[0] } + Write-Host "Detected dotnet SDKs: $($installedDotnetSdks -join ', ')" + foreach ($majorMinorVersion in $DotnetSDKVersionRequirements.Keys) { + $minimalVersion = "$majorMinorVersion.$($DotnetSDKVersionRequirements[$majorMinorVersion].MinimalPatch)" + $firstAcceptable = $installedDotnetSdks | + Where-Object { $_.StartsWith("$majorMinorVersion.") } | + Where-Object { [System.Management.Automation.SemanticVersion]::new($_) -ge [System.Management.Automation.SemanticVersion]::new($minimalVersion) } | + Select-Object -First 1 + if (-not $firstAcceptable) { + throw "Cannot find the dotnet SDK for .NET Core $majorMinorVersion. Version $minimalVersion or higher is required. Please specify '-Bootstrap' to install build dependencies." + } + } +} + +function Install-Dotnet { + [CmdletBinding()] + param( + [string]$Channel = 'release' + ) + try { + Find-Dotnet + return # Simply return if we find dotnet SDk with the correct version + } catch { } + $obtainUrl = "https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain" + try { + $installScript = if ($IsWindows) { "dotnet-install.ps1" } else { "dotnet-install.sh" } + Invoke-WebRequest -Uri $obtainUrl/$installScript -OutFile $installScript + foreach ($majorMinorVersion in $DotnetSDKVersionRequirements.Keys) { + $version = "$majorMinorVersion.$($DotnetSDKVersionRequirements[$majorMinorVersion].DefaultPatch)" + Write-Host "Installing dotnet SDK version $version" + if ($IsWindows) { + & .\$installScript -InstallDir "$env:ProgramFiles/dotnet" -Channel $Channel -Version $Version + } else { + bash ./$installScript --install-dir "/usr/share/dotnet" -c $Channel -v $Version + } + } + AddLocalDotnetDirPath + } + finally { + Remove-Item $installScript -Force -ErrorAction SilentlyContinue + } +} \ No newline at end of file From 22ce34f1e7435dc57689bbf7db8bb184c20eee02 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 14:53:36 -0700 Subject: [PATCH 29/31] Correct spacing issue --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9dc3928..c31b52f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -46,9 +46,9 @@ steps: SimulateReleaseBuild: $(SimulateReleaseBuild) - pwsh: | - Import-Module ".\pipelineUtilities.psm1" -Force - Install-Dotnet - displayName: 'Install .NET 3.1' + Import-Module ".\pipelineUtilities.psm1" -Force + Install-Dotnet + displayName: 'Install .NET 3.1' - pwsh: | Write-Host "IsReleaseBuild set to $env:IsReleaseBuild" From dbc033dd70493f4585c7595aabeafd851e096453 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 15:03:25 -0700 Subject: [PATCH 30/31] Add back suborchestrator test --- azure-pipelines.yml | 2 + .../DurableClientTests.cs | 66 +++++++++---------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c31b52f..2dab0ac 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,6 +81,7 @@ steps: archiveType: 'tar' archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName).tar.gz' replaceExistingArchive: true + displayName: 'Tar build tartifacts' - task: PublishBuildArtifacts@1 displayName: 'Publish .tar artifact' @@ -88,6 +89,7 @@ steps: PathtoPublish: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifactName).tar.gz condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) + displayName: 'Publish build artifacts' - task: PublishTestResults@2 inputs: diff --git a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs index 2d2cd95..c18f9d1 100644 --- a/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs +++ b/test/E2E/AzureFunctions.PowerShell.Durable.SDK.E2E/DurableClientTests.cs @@ -51,41 +51,41 @@ await ValidateDurableWorkflowResults( }); } - // [Fact] - // public async Task DurableSubOrchestratorCompletes() - // { - // var initialResponse = await Utilities.GetHttpStartResponse("SubOrchestrator"); - // Assert.Equal(HttpStatusCode.Accepted, initialResponse.StatusCode); + [Fact] + public async Task DurableSubOrchestratorCompletes() + { + var initialResponse = await Utilities.GetHttpStartResponse("SubOrchestrator"); + Assert.Equal(HttpStatusCode.Accepted, initialResponse.StatusCode); - // var location = initialResponse.Headers.Location; - // Assert.NotNull(location); + var location = initialResponse.Headers.Location; + Assert.NotNull(location); - // await ValidateDurableWorkflowResults( - // initialResponse, - // validateInitialResponse: (dynamic initialStatusResponseBody) => - // { - // Assert.NotNull(initialStatusResponseBody.id); - // var statusQueryGetUri = (string)initialStatusResponseBody.statusQueryGetUri; - // Assert.Equal(location?.ToString(), statusQueryGetUri); - // Assert.NotNull(initialStatusResponseBody.sendEventPostUri); - // Assert.NotNull(initialStatusResponseBody.purgeHistoryDeleteUri); - // Assert.NotNull(initialStatusResponseBody.terminatePostUri); - // Assert.NotNull(initialStatusResponseBody.rewindPostUri); - // }, - // validateIntermediateResponse: (dynamic intermediateStatusResponseBody) => - // { - // var runtimeStatus = (string)intermediateStatusResponseBody.runtimeStatus; - // Assert.True( - // runtimeStatus == "Running" || runtimeStatus == "Pending", - // $"Unexpected runtime status: {runtimeStatus}"); - // }, - // validateFinalResponse: (dynamic finalStatusResponseBody) => - // { - // Assert.Equal("Completed", (string)finalStatusResponseBody.runtimeStatus); - // Assert.Equal("Hello Tokyo", finalStatusResponseBody.output[0].ToString()); - // Assert.Equal("Hello Seattle", finalStatusResponseBody.output[1].ToString()); - // }); - // } + await ValidateDurableWorkflowResults( + initialResponse, + validateInitialResponse: (dynamic initialStatusResponseBody) => + { + Assert.NotNull(initialStatusResponseBody.id); + var statusQueryGetUri = (string)initialStatusResponseBody.statusQueryGetUri; + Assert.Equal(location?.ToString(), statusQueryGetUri); + Assert.NotNull(initialStatusResponseBody.sendEventPostUri); + Assert.NotNull(initialStatusResponseBody.purgeHistoryDeleteUri); + Assert.NotNull(initialStatusResponseBody.terminatePostUri); + Assert.NotNull(initialStatusResponseBody.rewindPostUri); + }, + validateIntermediateResponse: (dynamic intermediateStatusResponseBody) => + { + var runtimeStatus = (string)intermediateStatusResponseBody.runtimeStatus; + Assert.True( + runtimeStatus == "Running" || runtimeStatus == "Pending", + $"Unexpected runtime status: {runtimeStatus}"); + }, + validateFinalResponse: (dynamic finalStatusResponseBody) => + { + Assert.Equal("Completed", (string)finalStatusResponseBody.runtimeStatus); + Assert.Equal("Hello Tokyo", finalStatusResponseBody.output[0].ToString()); + Assert.Equal("Hello Seattle", finalStatusResponseBody.output[1].ToString()); + }); + } [Fact] public async Task DurableClientTerminatesOrchestration() From 03dc78a5d361867901d3e45cae6c3cc3ae6a0605 Mon Sep 17 00:00:00 2001 From: Michael Peng Date: Fri, 5 May 2023 15:07:17 -0700 Subject: [PATCH 31/31] Remove redundant field --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2dab0ac..aacfbd6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,7 +84,6 @@ steps: displayName: 'Tar build tartifacts' - task: PublishBuildArtifacts@1 - displayName: 'Publish .tar artifact' inputs: PathtoPublish: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifactName).tar.gz