diff --git a/.gitignore b/.gitignore index 3cf8180..05418c1 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,8 @@ local.settings.json # Temporary output folder for Durable SDK binaries **/src/out +# Package folder for .tar file +/package/** + # VS publish settings *.pubxml \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2d3664a..aacfbd6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,12 +18,78 @@ pool: demands: - ImageOverride -equals $(imageName) +variables: + artifactName: 'azure-functions-durable-powershell-$(Build.SourceVersion)' + # Every build will increment + buildNumber: $[counter('build', 001) ] + modulePath: './test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' + steps: -- pwsh: ./test/E2E/Start-E2ETest.ps1 -UseCoreToolsBuildFromIntegrationTests +- pwsh: | + $simulateReleaseBuild = $null + Write-Host "SimulateReleaseBuild set to $env:SimulateReleaseBuild" + if (-not([bool]::TryParse($env:SimulateReleaseBuild, [ref] $simulateReleaseBuild))) + { + throw "SimulateReleaseBuild can only be set to true or false." + } + + $isReleaseBuild = $false + if ($env:BuildSourceBranchName -like "release_*" -or $simulateReleaseBuild) + { + $isReleaseBuild = $true + } + Write-Host "Setting IsReleaseBuild to $isReleaseBuild because SimulateReleaseBuild is $env:SimulateReleaseBuild" + Write-Host "##vso[task.setvariable variable=IsReleaseBuild]$isReleaseBuild" + Write-Host "IsReleaseBuild: $isReleaseBuild" + displayName: Set IsReleaseBuild pipeline variable + env: + SimulateReleaseBuild: $(SimulateReleaseBuild) + +- pwsh: | + Import-Module ".\pipelineUtilities.psm1" -Force + Install-Dotnet + displayName: 'Install .NET 3.1' + +- pwsh: | + Write-Host "IsReleaseBuild set to $env:IsReleaseBuild" + $isReleaseBuild = $false + if (-not([bool]::TryParse($env:IsReleaseBuild, [ref] $isReleaseBuild))) + { + throw "SimulateReleaseBuild can only be set to true or false." + } + + # We only generate an SBOM for release or simulated release builds + Write-Host "Running ./build.ps1 -Configuration Release -AddSBOM:$isReleaseBuild..." + ./build.ps1 -Configuration Release -AddSBOM:$isReleaseBuild + displayName: 'Build Durable SDK' + env: + # We include IsReleaseBuild as an environment variable since Linux agents don't seem to support including + # pipeline variables in scripts with the $(variable) syntax + IsReleaseBuild: $(IsReleaseBuild) + SBOMUtilSASUrl: $(SBOMUtilSASUrl) + +- pwsh: | + ./test/E2E/Start-E2ETest.ps1 -NoBuild -UseCoreToolsBuildFromIntegrationTests env: AzureWebJobsStorage: $(AzureWebJobsStorage) displayName: 'Run E2E tests' +- task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(Build.SourcesDirectory)/test/E2E/durableApp/Modules/AzureFunctions.PowerShell.Durable.SDK' + includeRootFolder: false + archiveType: 'tar' + archiveFile: '$(Build.ArtifactStagingDirectory)/$(artifactName).tar.gz' + replaceExistingArchive: true + displayName: 'Tar build tartifacts' + +- task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifactName).tar.gz + condition: and(succeeded(), eq(variables['IsReleaseBuild'], 'true')) + displayName: 'Publish build artifacts' + - task: PublishTestResults@2 inputs: testResultsFormat: 'VSTest' diff --git a/build.ps1 b/build.ps1 index 71dc996..74ae193 100644 --- a/build.ps1 +++ b/build.ps1 @@ -3,9 +3,13 @@ param( [ValidateSet('Debug', 'Release')] [string] - $Configuration = 'Debug' + $Configuration = 'Debug', + [switch] + $AddSBOM ) +Import-Module "$PSScriptRoot\pipelineUtilities.psm1" -Force + $packageName = "AzureFunctions.PowerShell.Durable.SDK" $shimPath = "$PSScriptRoot/src/DurableSDK" $durableEnginePath = "$PSScriptRoot/src/DurableEngine" @@ -13,46 +17,15 @@ $durableAppPath = "$PSScriptRoot/test/E2E/durableApp/Modules/$packageName" $powerShellModulePath = "$PSScriptRoot/src/$packageName.psm1" $manifestPath = "$PSScriptRoot/src/$packageName.psd1" -$outputPath = "$PSScriptRoot/src/out/" -if ($Configuration -eq "Debug") -{ - # Publish directly to the test durable app for testing - $outputPath = $durableAppPath -} +# Publish directly to the test durable app for testing +$outputPath = $durableAppPath + $sharedDependenciesPath = "$outputPath/Dependencies/" $netCoreTFM = 'net6.0' $publishPathSuffix = "bin/$Configuration/$netCoreTFM/publish" -function Write-Log -{ - param ( - [Parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - [System.String] - $Message, - - [Switch] - $Warning, - - [Switch] - $Throw, - - [System.String] - $Color - ) - - $Message = (Get-Date -Format G) + " -- $Message" - - if ($Throw) - { - throw $Message - } - - $foregroundColor = if ($Warning.IsPresent) { 'Yellow' } elseif ($Color) { $Color } else { 'Green' } - Write-Host -ForegroundColor $foregroundColor $Message -} - +#region BUILD ARTIFACTS =========================================================================== Write-Log "Build started..." Write-Log "Configuration: '$Configuration'`nOutput folder '$outputPath'`nShared dependencies folder: '$sharedDependenciesPath'" "Gray" @@ -89,13 +62,13 @@ foreach ($project in $projects.GetEnumerator()) { $commonFiles = [System.Collections.Generic.HashSet[string]]::new() Write-Log "Copying assemblies from the Durable Engine project into $sharedDependenciesPath" "Gray" -Get-ChildItem -Path "$durableEnginePath/$publishPathSuffix" | +Get-ChildItem -Path (Join-Path "$durableEnginePath" "$publishPathSuffix") | Where-Object { $_.Extension -in '.dll','.pdb' } | ForEach-Object { [void]$commonFiles.Add($_.Name); Copy-Item -LiteralPath $_.FullName -Destination $sharedDependenciesPath } # Copy all *unique* assemblies from Durable SDK into output directory Write-Log "Copying unique assemblies from the Durable SDK project into $outputPath" "Gray" -Get-ChildItem -Path "$shimPath/$publishPathSuffix" | +Get-ChildItem -Path (Join-Path "$shimPath" "$publishPathSuffix") | Where-Object { $_.Extension -in '.dll','.pdb' -and -not $commonFiles.Contains($_.Name) } | ForEach-Object { Copy-Item -LiteralPath $_.FullName -Destination $outputPath } @@ -103,4 +76,23 @@ Get-ChildItem -Path "$shimPath/$publishPathSuffix" | Write-Log "Copying PowerShell module and manifest from the Durable SDK source code into $outputPath" "Gray" Copy-Item -Path $powerShellModulePath -Destination $outputPath Copy-Item -Path $manifestPath -Destination $outputPath -Write-Log "Build succeeded!" \ No newline at end of file +Write-Log "Build succeeded!" +#endregion + +#region ADD SBOM ================================================================================== +if ($AddSBOM) { + # Install manifest tool + $manifestToolPath = Install-SBOMUtil + Write-Log "Manifest tool path: $manifestToolPath" + + # Generate manifest + $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json") + $packageName = "AzureFunctions.PowerShell.Durable.SDK" + + Write-Log "Running: dotnet $manifestToolPath generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName" + dotnet $manifestToolPath generate -BuildDropPath $outputPath -BuildComponentPath $outputPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName + + # Discard telemetry generated + Remove-Item -Path $telemetryFilePath -ErrorAction Ignore +} +#endregion \ No newline at end of file diff --git a/pipelineUtilities.psm1 b/pipelineUtilities.psm1 new file mode 100644 index 0000000..f2332f9 --- /dev/null +++ b/pipelineUtilities.psm1 @@ -0,0 +1,128 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +using namespace System.Runtime.InteropServices + +$DotnetSDKVersionRequirements = @{ + + # .NET SDK 3.1 is required by the Microsoft.ManifestTool.dll tool + '3.1' = @{ + MinimalPatch = '415' + DefaultPatch = '415' + } +} + +function Write-Log +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [System.String] + $Message, + + [Switch] + $Warning, + + [Switch] + $Throw, + + [System.String] + $Color + ) + + $Message = (Get-Date -Format G) + " -- $Message" + + if ($Throw) + { + throw $Message + } + + $foregroundColor = if ($Warning.IsPresent) { 'Yellow' } elseif ($Color) { $Color } else { 'Green' } + Write-Host -ForegroundColor $foregroundColor $Message +} + +function Install-SBOMUtil +{ + if ([string]::IsNullOrEmpty($env:SBOMUtilSASUrl)) + { + throw "The `$SBOMUtilSASUrl environment variable cannot be null or empty when specifying the `$AddSBOM switch" + } + + $MANIFESTOOLNAME = "ManifestTool" + Write-Log "Installing $MANIFESTOOLNAME..." + + $MANIFESTOOL_DIRECTORY = Join-Path $PSScriptRoot $MANIFESTOOLNAME + Remove-Item -Recurse -Force $MANIFESTOOL_DIRECTORY -ErrorAction Ignore + + Invoke-RestMethod -Uri $env:SBOMUtilSASUrl -OutFile "$MANIFESTOOL_DIRECTORY.zip" + Expand-Archive "$MANIFESTOOL_DIRECTORY.zip" -DestinationPath $MANIFESTOOL_DIRECTORY + + $dllName = "Microsoft.ManifestTool.dll" + $manifestToolPath = Join-Path "$MANIFESTOOL_DIRECTORY" "$dllName" + + if (-not (Test-Path $manifestToolPath)) + { + throw "$MANIFESTOOL_DIRECTORY does not contain '$dllName'" + } + + Write-Log 'Done.' + + return $manifestToolPath +} + + +function AddLocalDotnetDirPath { + $LocalDotnetDirPath = if ($IsWindows) { "$env:ProgramFiles/dotnet" } else { "/usr/share/dotnet" } + if (($env:PATH -split [IO.Path]::PathSeparator) -notcontains $LocalDotnetDirPath) { + $env:PATH = $LocalDotnetDirPath + [IO.Path]::PathSeparator + $env:PATH + } +} + +function Find-Dotnet +{ + AddLocalDotnetDirPath + $listSdksOutput = dotnet --list-sdks + $installedDotnetSdks = $listSdksOutput | ForEach-Object { $_.Split(" ")[0] } + Write-Host "Detected dotnet SDKs: $($installedDotnetSdks -join ', ')" + foreach ($majorMinorVersion in $DotnetSDKVersionRequirements.Keys) { + $minimalVersion = "$majorMinorVersion.$($DotnetSDKVersionRequirements[$majorMinorVersion].MinimalPatch)" + $firstAcceptable = $installedDotnetSdks | + Where-Object { $_.StartsWith("$majorMinorVersion.") } | + Where-Object { [System.Management.Automation.SemanticVersion]::new($_) -ge [System.Management.Automation.SemanticVersion]::new($minimalVersion) } | + Select-Object -First 1 + if (-not $firstAcceptable) { + throw "Cannot find the dotnet SDK for .NET Core $majorMinorVersion. Version $minimalVersion or higher is required. Please specify '-Bootstrap' to install build dependencies." + } + } +} + +function Install-Dotnet { + [CmdletBinding()] + param( + [string]$Channel = 'release' + ) + try { + Find-Dotnet + return # Simply return if we find dotnet SDk with the correct version + } catch { } + $obtainUrl = "https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain" + try { + $installScript = if ($IsWindows) { "dotnet-install.ps1" } else { "dotnet-install.sh" } + Invoke-WebRequest -Uri $obtainUrl/$installScript -OutFile $installScript + foreach ($majorMinorVersion in $DotnetSDKVersionRequirements.Keys) { + $version = "$majorMinorVersion.$($DotnetSDKVersionRequirements[$majorMinorVersion].DefaultPatch)" + Write-Host "Installing dotnet SDK version $version" + if ($IsWindows) { + & .\$installScript -InstallDir "$env:ProgramFiles/dotnet" -Channel $Channel -Version $Version + } else { + bash ./$installScript --install-dir "/usr/share/dotnet" -c $Channel -v $Version + } + } + AddLocalDotnetDirPath + } + finally { + Remove-Item $installScript -Force -ErrorAction SilentlyContinue + } +} \ No newline at end of file diff --git a/src/DurableSDK/DurableSDK.csproj b/src/DurableSDK/DurableSDK.csproj index 9e3b3b5..ed4cd9f 100644 --- a/src/DurableSDK/DurableSDK.csproj +++ b/src/DurableSDK/DurableSDK.csproj @@ -8,6 +8,6 @@ - + diff --git a/test/E2E/Start-E2ETest.ps1 b/test/E2E/Start-E2ETest.ps1 index 7136b3a..3210538 100644 --- a/test/E2E/Start-E2ETest.ps1 +++ b/test/E2E/Start-E2ETest.ps1 @@ -4,6 +4,8 @@ # param ( + [Switch] + $NoBuild, [Switch] $UseCoreToolsBuildFromIntegrationTests, [Switch] @@ -98,11 +100,11 @@ Write-Host "Deleting $FUNC_CLI_DIRECTORY if it exists..." Remove-Item -Force "$FUNC_CLI_DIRECTORY.zip" -ErrorAction Ignore Remove-Item -Recurse -Force $FUNC_CLI_DIRECTORY -ErrorAction Ignore -if (-not $SkipCoreToolsDownload.IsPresent) +if (-not $SkipCoreToolsDownload) { Write-Host "Downloading Core Tools because SkipCoreToolsDownload switch parameter is not present..." $coreToolsDownloadURL = $null - if ($UseCoreToolsBuildFromIntegrationTests.IsPresent) + if ($UseCoreToolsBuildFromIntegrationTests) { $coreToolsDownloadURL = "https://functionsintegclibuilds.blob.core.windows.net/builds/$FUNC_RUNTIME_VERSION/latest/Azure.Functions.Cli.$os-$arch.zip" $env:CORE_TOOLS_URL = "https://functionsintegclibuilds.blob.core.windows.net/builds/$FUNC_RUNTIME_VERSION/latest" @@ -148,16 +150,17 @@ if (-not $SkipCoreToolsDownload.IsPresent) $env:FUNC_PATH = $funcPath Write-Host "Set FUNC_PATH environment variable to $env:FUNC_PATH" -# For both integration build test runs and regular test runs, we copy binaries to durableApp/Modules -Write-Host "Building the DurableSDK module and copying binaries to the durableApp/Modules directory..." -$configuration = if ($env:CONFIGURATION) { $env:CONFIGURATION } else { 'Debug' } - -Push-Location "$PSScriptRoot/../.." -try { - & ./build.ps1 -Configuration 'Debug' -} -finally { - Pop-Location +if (-not $NoBuild) { + # For both integration build test runs and regular test runs, we copy binaries to durableApp/Modules + Write-Host "Building the DurableSDK module and copying binaries to the durableApp/Modules directory..." + + Push-Location "$PSScriptRoot/../.." + try { + & ./build.ps1 -Configuration 'Debug' + } + finally { + Pop-Location + } } Write-Host "Starting Core Tools..." diff --git a/test/E2E/durableApp/SimpleOrchestrator/run.ps1 b/test/E2E/durableApp/SimpleOrchestrator/run.ps1 index dca78f0..19c4ae4 100644 --- a/test/E2E/durableApp/SimpleOrchestrator/run.ps1 +++ b/test/E2E/durableApp/SimpleOrchestrator/run.ps1 @@ -6,4 +6,4 @@ $ErrorActionPreference = 'Stop' $output = Invoke-DurableActivity -FunctionName "Hello" -Input "Tokyo" -return $output +$output