diff --git a/.pipelines/DSC-Official.yml b/.pipelines/DSC-Official.yml index f99834119..4d9cf7259 100644 --- a/.pipelines/DSC-Official.yml +++ b/.pipelines/DSC-Official.yml @@ -7,19 +7,25 @@ pr: - onebranch - release/v* -schedules: -- cron: '0 3 * * 1' - displayName: Weekly Build - branches: - include: - - main - always: true +parameters: + - name: OfficialBuild + type: boolean + default: false variables: - BuildConfiguration: 'release' - PackageRoot: '$(System.ArtifactsDirectory)/Packages' - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - Codeql.TSAEnabled: true + - name: BuildConfiguration + value: 'release' + - name: PackageRoot + value: '$(System.ArtifactsDirectory)/Packages' + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + - name: Codeql.TSAEnabled + value: true + - name: templateFile + value: ${{ iif ( parameters.OfficialBuild, 'v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates', 'v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates' ) }} + - group: DSC-Rust.SDK + - name: officialBuild + value: ${{ parameters.OfficialBuild }} resources: repositories: @@ -29,8 +35,10 @@ resources: ref: refs/heads/main extends: - template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates + template: ${{ variables.templateFile }} parameters: + release: + category: NonAzure featureFlags: WindowsHostVersion: Disk: Large @@ -83,6 +91,18 @@ extends: Write-Host ("sending " + $vstsCommandString) Write-Host "##$vstsCommandString" name: Package + displayName: Set Package Version + - task: AzureCLI@2 + displayName: Get Az Token + inputs: + azureSubscription: PowerShell-CICD-Feed-Access + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $token = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv + $vstsCommandString = "vso[task.setvariable variable=AzToken;isoutput=true]$token" + Write-Host "Setting token" + Write-Host "##$vstsCommandString" - job: BuildWin_x64 dependsOn: SetPackageVersion @@ -90,6 +110,7 @@ extends: ob_sdl_tsa_configFile: '$(Build.SourcesDirectory)\DSC\.config\tsaoptions.json' ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' signSrcPath: '$(Build.SourcesDirectory)\out' + AzToken: $[ dependencies.SetPackageVersion.outputs['AzToken'] ] ob_sdl_sbom_enabled: true ob_signing_setup_enabled: true ob_sdl_codeql_compiled_enabled: true @@ -101,6 +122,8 @@ extends: buildName: x86_64-pc-windows-msvc signSrcPath: '$(signSrcPath)' PackageRoot: '$(PackageRoot)' + aztoken: '$(AzToken)' + rustSDK: '$(Rust.SDK)' - job: BuildWin_arm64 dependsOn: SetPackageVersion @@ -108,6 +131,7 @@ extends: ob_sdl_tsa_configFile: '$(Build.SourcesDirectory)\DSC\.config\tsaoptions.json' ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' signSrcPath: '$(Build.SourcesDirectory)\out' + AzToken: $[ dependencies.SetPackageVersion.outputs['AzToken'] ] ob_sdl_sbom_enabled: true ob_signing_setup_enabled: true ob_sdl_codeql_compiled_enabled: true @@ -119,6 +143,8 @@ extends: buildName: aarch64-pc-windows-msvc signSrcPath: '$(signSrcPath)' PackageRoot: '$(PackageRoot)' + aztoken: '$(AzToken)' + rustSDK: '$(Rust.SDK)' - job: CreateMsixBundle dependsOn: @@ -127,9 +153,14 @@ extends: variables: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' ob_sdl_tsa_configFile: '$(Build.SourcesDirectory)\DSC\.config\tsaoptions.json' - ob_sdl_sbom_enabled: true - ob_signing_setup_enabled: true - ob_sdl_codeql_compiled_enabled: true + ob_symbolsPublishing_enabled: true + ob_symbolsPublishing_symbolsFolder: '$(Build.SourcesDirectory)\DSC\bin' + ob_symbolsPublishing_searchPattern: '**/*.pdb' + ob_symbolsPublishing_indexSources: true + ob_sdl_sbom_enabled: false + ob_signing_setup_enabled: false + ob_sdl_codeql_compiled_enabled: false + ob_restore_phase: true pool: type: windows steps: @@ -146,16 +177,51 @@ extends: Copy-Item "$(Pipeline.Workspace)/drop_BuildAndSign_BuildWin_x64/*.msix" ./bin/msix -Verbose Copy-Item "$(Pipeline.Workspace)/drop_BuildAndSign_BuildWin_arm64/*.msix" ./bin/msix -Verbose ./build.ps1 -PackageType msixbundle - Copy-Item ./bin/*.msixbundle "$(ob_outputDirectory)" + Copy-Item "$(Build.SourcesDirectory)/DSC/bin/*.msixbundle" "$(ob_outputDirectory)" displayName: 'Create msixbundle' condition: succeeded() + - job: SignMsixBundle + condition: and(succeeded(), eq(variables.officialBuild, true)) + dependsOn: CreateMsixBundle + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + ob_sdl_sbom_enabled: false + ob_signing_setup_enabled: true + ob_sdl_codeql_compiled_enabled: false + pool: + type: windows + steps: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + artifact: drop_BuildAndSign_CreateMsixBundle + itemPattern: | + **/*.msixbundle + targetPath: '$(Build.ArtifactStagingDirectory)/downloads' + displayName: Download MsixBundle + - task: onebranch.pipeline.signing@1 + displayName: Sign MsixBundle + condition: succeeded() + inputs: + command: 'sign' + signing_profile: 'Dynamic-WINMSAPP1ST' + files_to_sign: '*.msixbundle' + search_root: '$(Build.ArtifactStagingDirectory)/downloads' + - pwsh: | + Copy-Item "$(Build.ArtifactStagingDirectory)/downloads/*.msixbundle" "$(ob_outputDirectory)" + displayName: 'Copy MsixBundle to output directory' + - job: BuildLinuxMusl dependsOn: SetPackageVersion variables: LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ] + AzToken: $[ dependencies.SetPackageVersion.outputs['AzToken'] ] ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + ob_linuxSymbolsPublishing_enabled: true + ob_linuxSymbolsPublishing_symbolsFolder: '$(Build.SourcesDirectory)/DSC/bin' + ob_linuxSymbolsPublishing_searchPattern: '**/*.dbg' displayName: Linux-x64-musl pool: type: linux @@ -163,7 +229,7 @@ extends: - task: RustInstaller@1 inputs: rustVersion: ms-stable - toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json + toolchainFeed: $(Rust.SDK) additionalTargets: x86_64-unknown-linux-musl displayName: Install Rust env: @@ -171,6 +237,9 @@ extends: - pwsh: | apt update apt -y install musl-tools + $header = "Bearer $(AzToken)" + $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header + $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token' ./build.ps1 -Release -Architecture x86_64-unknown-linux-musl ./build.ps1 -PackageType tgz -Architecture x86_64-unknown-linux-musl -Release Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)" @@ -180,9 +249,13 @@ extends: - job: BuildLinuxArm64Musl dependsOn: SetPackageVersion variables: - LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2004-arm64:latest' + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204-arm64:latest' PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ] + AzToken: $[ dependencies.SetPackageVersion.outputs['AzToken'] ] ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + ob_linuxSymbolsPublishing_enabled: true + ob_linuxSymbolsPublishing_symbolsFolder: '$(Build.SourcesDirectory)/DSC/bin' + ob_linuxSymbolsPublishing_searchPattern: '**/*.dbg' displayName: Linux-ARM64-musl pool: type: linux @@ -191,11 +264,19 @@ extends: - task: RustInstaller@1 inputs: rustVersion: ms-stable - toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json + toolchainFeed: $(Rust.SDK) additionalTargets: aarch64-unknown-linux-musl displayName: Install Rust env: ob_restore_phase: true + - task: AzureCLI@2 + displayName: Azure CLI + inputs: + azureSubscription: PowerShell-CICD-Feed-Access + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az account show - pwsh: | $env:CC_aarch64_unknown_linux_musl='clang' $env:AR_aarch64_unknown_linux_musl='llvm-ar' @@ -204,13 +285,17 @@ extends: apt -y install clang apt -y install llvm apt -y install musl-tools - apt -y install gcc-multilib + apt -y install musl-gcc + #apt -y install gcc-multilib apt -y install libssl-dev apt -y install pkg-config msrustup default stable-aarch64-unknown-linux-musl if ((openssl version -d) -match 'OPENSSLDIR: "(?.*?)"') { $env:OPENSSL_LIB_DIR = $matches['dir'] } + $header = "Bearer $(AzToken)" + $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header + $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token' ./build.ps1 -Release -Architecture aarch64-unknown-linux-musl ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-musl -Release Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)" @@ -221,6 +306,7 @@ extends: dependsOn: SetPackageVersion variables: PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ] + AzToken: $[ dependencies.SetPackageVersion.outputs['AzToken'] ] ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' displayName: BuildMac pool: @@ -238,12 +324,24 @@ extends: - task: RustInstaller@1 inputs: rustVersion: ms-stable - toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json + toolchainFeed: $(Rust.SDK) additionalTargets: $(buildName) displayName: Install Rust env: ob_restore_phase: true + - task: AzureCLI@2 + displayName: Azure CLI + inputs: + azureSubscription: PowerShell-CICD-Feed-Access + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az account show - pwsh: | + $header = "Bearer $(AzToken)" + $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header + $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token' + Write-Verbose -Verbose "Building for $(buildName)" ./build.ps1 -Release -Architecture $(buildName) ./build.ps1 -PackageType tgz -Architecture $(buildName) -Release Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)" @@ -251,53 +349,258 @@ extends: displayName: 'Build $(buildName)' condition: succeeded() - - stage: Release + - stage: ReleasePreparation dependsOn: BuildAndSign - condition: ne(variables['Build.Reason'], 'Schedule') - variables: - PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ] + condition: and(succeeded(), ne(variables['Build.Reason'], 'Schedule'), eq(variables.officialBuild, true)) jobs: - - job: Validation - displayName: Manual validation + - job: ReleasePreparationJob + displayName: Release Preparation job + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - download: current + artifact: drop_BuildAndSign_BuildWin_x64 + patterns: '*.zip' + + - download: current + artifact: drop_BuildAndSign_BuildWin_arm64 + patterns: '*.zip' + + - download: current + artifact: drop_BuildAndSign_BuildLinuxArm64Musl + patterns: '*.tar.gz' + + - download: current + artifact: drop_BuildAndSign_BuildLinuxMusl + patterns: '*.tar.gz' + + - download: current + artifact: release ## this includes artifacts for macOS + patterns: '**/*.tar.gz' + + - download: current + artifact: drop_BuildAndSign_SignMsixBundle + patterns: '*.msixbundle' + + - pwsh: | + Get-ChildItem "$(Pipeline.Workspace)" -Recurse -Include '*.zip', '*.tar.gz', '*.msixbundle' | ForEach-Object { + Write-Host "Found artifact: $($_.FullName)" + } + displayName: List downloaded artifacts + + - pwsh: | + $outputDir = "$(ob_outputDirectory)\releasePrep" + if( -not (Test-Path -Path $outputDir)) { + New-Item -ItemType Directory -Path $outputDir -Force -ErrorAction Ignore -Verbose + } + + Write-Verbose -Verbose "Starting to copy" + + Get-ChildItem "$(Pipeline.Workspace)" -Recurse -Include '*.zip', '*.tar.gz', '*.msixbundle' | ForEach-Object { + Copy-Item -Path $_.FullName -Destination $outputDir -Force -Verbose + } + + Write-Verbose -Verbose "Copy completed" + displayName: Copy artifacts to release area + + - stage: Validation + dependsOn: ['ReleasePreparation'] + condition: and(succeeded(), ne(variables['Build.Reason'], 'Schedule')) + jobs: + - job: ValidationJob + displayName: Validation job pool: type: agentless - timeoutInMinutes: 1440 steps: - task: ManualValidation@0 - displayName: Wait 24 hours for validation + displayName: Wait for manual validation inputs: notifyUsers: $(Build.RequestedForEmail) - instructions: Please validate the release + instructions: Please validate the build artifacts before proceeding to release. timeoutInMinutes: 1440 - - job: GitHub - dependsOn: validation - displayName: Publish draft to GitHub + + - stage: Release + dependsOn: ['BuildAndSign', 'Validation', 'ReleasePreparation'] + condition: and(succeeded(), ne(variables['Build.Reason'], 'Schedule'), eq(variables.officialBuild, true)) + variables: + - name: PackageVersion + value: $[ stageDependencies.BuildAndSign.SetPackageVersion.outputs['Package.Version'] ] + - name: ob_release_environment + value: ${{ iif ( parameters.OfficialBuild, 'Production', 'Test' ) }} + jobs: + - job: ReleaseJob + displayName: Release job pool: - type: windows - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - ob_sdl_sbom_enabled: false - ob_signing_setup_enabled: false - ob_sdl_codeql_compiled_enabled: false - drop: $(Pipeline.Workspace)/drop_build_main + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + artifactName: drop_ReleasePreparation_ReleasePreparationJob + steps: - - download: current - displayName: Download artifacts - patterns: | - '**/*.zip' - '**/*.tar.gz' + - task: PowerShell@2 + displayName: Capture downloaded artifacts and Version for release + inputs: + targetType: 'inline' + script: | + Write-Verbose -Verbose "Release version: $(PackageVersion)" + + $artifacts = Get-ChildItem "$(Pipeline.Workspace)" -Recurse -Include '*.zip', '*.tar.gz', '*.msixbundle' + + $artifacts | ForEach-Object { + Write-Verbose -Verbose "Found artifact: $($_.FullName)" + } + + $GitHubReleaseDirectory = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/GitHubRelease" -Force -ErrorAction Ignore + Write-Host "##vso[task.setvariable variable=GitHubReleaseDirectory]$GitHubReleaseDirectory" + $artifacts | ForEach-Object { + Copy-Item -Path $_.FullName -Destination $GitHubReleaseDirectory -Force -Verbose + } + + if (-not '$(PackageVersion)') { + throw "PackageVersion variable is not set. Cannot proceed with release." + } + + $packageVersion = '$(PackageVersion)' + if ($packageVersion -like '*-*') { + Write-Verbose -Verbose "Pre-release version detected: $packageVersion" + Write-Host "##vso[task.setvariable variable=IsPreRelease]true" + } + else { + Write-Verbose -Verbose "Stable release version detected: $packageVersion" + Write-Host "##vso[task.setvariable variable=IsPreRelease]false" + } + + $githubReleaseVersion = "v$packageVersion" + Write-Verbose -Verbose "GitHub Release version: $githubReleaseVersion" + Write-Host "##vso[task.setvariable variable=GitHubReleaseVersion]$githubReleaseVersion" + - task: GitHubRelease@1 displayName: Create GitHub release inputs: - gitHubConnection: GitHub + gitHubConnection: github.com_SteveL-MSFT repositoryName: PowerShell/DSC + target: main action: create assets: | - *.zip; - *.tar.gz; - addChangeLog: true - changeLogType: commitBased - releaseNotesFilePath: CHANGELOG.md - tagSource: gitTag - tag: v$(version) + $(GitHubReleaseDirectory)\*.zip + $(GitHubReleaseDirectory)\*.tar.gz + $(GitHubReleaseDirectory)\*.msixbundle + addChangeLog: false + tagSource: 'userSpecifiedTag' + tag: '$(GitHubReleaseVersion)' isDraft: true + isPreRelease: '$(IsPreRelease)' + + - stage: ReleaseUniversalPackage + dependsOn: ['BuildAndSign','Release'] + condition: and(succeeded(), ne(variables['Build.Reason'], 'Schedule'), eq(variables.officialBuild, true)) + variables: + - name: PackageVersion + value: $[ stageDependencies.BuildAndSign.SetPackageVersion.outputs['Package.Version'] ] + jobs: + - job: ReleaseUniversalPackageJob + displayName: Release Universal Package job + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - download: current + artifact: drop_ReleasePreparation_ReleasePreparationJob + + - pwsh: | + $releasePrepPath = "$(Pipeline.Workspace)/drop_ReleasePreparation_ReleasePreparationJob/releasePrep" + if (-not (Test-Path -Path $releasePrepPath)) { + throw "Release preparation path '$releasePrepPath' does not exist." + } + + $windowsFiles = Get-ChildItem -Path $releasePrepPath -Recurse -Include '*.zip' + if ($windowsFiles.Count -eq 0) { + throw "No Windows .zip files found in '$releasePrepPath'. Cannot proceed with Universal Package creation." + } + + $linuxFiles = Get-ChildItem -Path $releasePrepPath -Recurse -Include '*linux.tar.gz' + if ($linuxFiles.Count -eq 0) { + throw "No Linux .tar.gz files found in '$releasePrepPath'. Cannot proceed with Universal Package creation." + } + + $macosFiles = Get-ChildItem -Path $releasePrepPath -Recurse -Include '*darwin.tar.gz' + if ($macosFiles.Count -eq 0) { + throw "No macOS .tar.gz files found in '$releasePrepPath'. Cannot proceed with Universal Package creation." + } + + $windowsDirectory = New-Item -ItemType Directory -Path "$releasePrepPath/windows" -Force -ErrorAction Ignore + Write-Host "##vso[task.setvariable variable=WindowsDirectory]$($windowsDirectory.FullName)" + + $linuxDirectory = New-Item -ItemType Directory -Path "$releasePrepPath/linux" -Force -ErrorAction Ignore + Write-Host "##vso[task.setvariable variable=LinuxDirectory]$linuxDirectory" + + $macosDirectory = New-Item -ItemType Directory -Path "$releasePrepPath/macos" -Force -ErrorAction Ignore + Write-Host "##vso[task.setvariable variable=MacOSDirectory]$macosDirectory" + + $windowsFiles | ForEach-Object { + Move-Item -Path $_.FullName -Destination $windowsDirectory.FullName -Force -Verbose + } + + $linuxFiles | ForEach-Object { + Move-Item -Path $_.FullName -Destination $linuxDirectory.FullName -Force -Verbose + } + + $macosFiles | ForEach-Object { + Move-Item -Path $_.FullName -Destination $macosDirectory.FullName -Force -Verbose + } + + displayName: Prepare files for Universal Package + + - task: AzureCLI@2 + displayName: Publish Windows - Universal Package + inputs: + azureSubscription: PS-PS-DSC-UniversalFeed + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $packageVersion = '$(PackageVersion)' + + if (-not $packageVersion) { + throw "PackageVersion variable is not set. Cannot proceed with publishing Universal Package." + } + Write-Verbose -Verbose "Universal Package version: $packageVersion" + az artifacts universal publish --organization https://dev.azure.com/PowerShell --project PowerShell --feed PowerShell-Universal --name microsoft.dsc-windows --version $packageVersion --description "Microsoft Desired State Configuration (DSC) - Universal Package" --path "$(WindowsDirectory)" --scope project --verbose + condition: succeeded() + + - task: AzureCLI@2 + displayName: Publish Linux - Universal Package + inputs: + azureSubscription: PS-PS-DSC-UniversalFeed + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $packageVersion = '$(PackageVersion)' + + if (-not $packageVersion) { + throw "PackageVersion variable is not set. Cannot proceed with publishing Universal Package." + } + Write-Verbose -Verbose "Universal Package version: $packageVersion" + az artifacts universal publish --organization https://dev.azure.com//PowerShell --project PowerShell --feed PowerShell-Universal --name microsoft.dsc-linux --version $packageVersion --description "Microsoft Desired State Configuration (DSC) - Universal Package" --path "$(LinuxDirectory)" --scope project --verbose + condition: succeeded() + + - task: AzureCLI@2 + displayName: Publish macOS - Universal Package + inputs: + azureSubscription: PS-PS-DSC-UniversalFeed + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $packageVersion = '$(PackageVersion)' + + if (-not $packageVersion) { + throw "PackageVersion variable is not set. Cannot proceed with publishing Universal Package." + } + Write-Verbose -Verbose "Universal Package version: $packageVersion" + az artifacts universal publish --organization https://dev.azure.com/PowerShell/ --project PowerShell --feed PowerShell-Universal --name microsoft.dsc-macos --version $packageVersion --description "Microsoft Desired State Configuration (DSC) - Universal Package" --path "$(MacOSDirectory)" --scope project --verbose + condition: succeeded() + diff --git a/.pipelines/DSC-Windows.yml b/.pipelines/DSC-Windows.yml index 3a7ec1377..d7580ccae 100644 --- a/.pipelines/DSC-Windows.yml +++ b/.pipelines/DSC-Windows.yml @@ -8,6 +8,10 @@ parameters: - name: BuildConfiguration type: string default: Release + - name: aztoken + type: string + - name: RustSDK + type: string steps: - checkout: self @@ -29,12 +33,15 @@ steps: - task: RustInstaller@1 inputs: rustVersion: ms-stable - toolchainFeed: https://pkgs.dev.azure.com/mscodehub/Rust/_packaging/Rust/nuget/v3/index.json + toolchainFeed: ${{ parameters.RustSDK }} additionalTargets: ${{ parameters.buildName }} displayName: Install Rust env: ob_restore_phase: true - pwsh: | + $header = "Bearer ${{ parameters.aztoken }}" + $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header + $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token' Set-Location "$(Build.SourcesDirectory)/DSC" $LLVMBIN = "$($env:PROGRAMFILES)\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm\bin" if (!(Test-Path $LLVMBIN)) { @@ -42,6 +49,7 @@ steps: } $env:PATH += ";$LLVMBIN" write-verbose -verbose (gcm clang.exe | out-string) + Write-Verbose -Verbose "Building for ${{ parameters.buildName }}" ./build.ps1 -Release -Architecture ${{ parameters.buildName }} -SkipLinkCheck displayName: 'Build ${{ parameters.buildName }}' env: diff --git a/build.ps1 b/build.ps1 index f6f9adab4..ad8e86a32 100755 --- a/build.ps1 +++ b/build.ps1 @@ -15,7 +15,7 @@ param( [switch]$GetPackageVersion, [switch]$SkipLinkCheck, [switch]$UseX64MakeAppx, - [switch]$UseCratesIO, + [switch]$UseCFS, [switch]$UpdateLockFile, [switch]$Audit, [switch]$UseCFSAuth, @@ -23,7 +23,18 @@ param( [switch]$Verbose ) +trap { + Write-Error "An error occurred: $($_ | Out-String)" + exit 1 +} + $env:RUSTC_LOG=$null +$env:RUSTFLAGS='-Dwarnings' +$usingADO = ($null -ne $env:TF_BUILD) +if ($usingADO -or $UseCFSAuth) { + $UseCFS = $true +} + if ($Verbose) { $env:RUSTC_LOG='rustc_codegen_ssa::back::link=info' } @@ -129,6 +140,7 @@ function Find-LinkExe { $channel = 'stable' if ($null -ne (Get-Command msrustup -CommandType Application -ErrorAction Ignore)) { + Write-Verbose -Verbose "Using msrustup" $rustup = 'msrustup' $channel = 'ms-stable' if ($architecture -eq 'current') { @@ -136,15 +148,46 @@ if ($null -ne (Get-Command msrustup -CommandType Application -ErrorAction Ignore } } elseif ($null -ne (Get-Command rustup -CommandType Application -ErrorAction Ignore)) { $rustup = 'rustup' -} else { - $rustup = 'echo' } if ($null -ne $packageType) { $SkipBuild = $true } else { + if ($UseCFS) { + Write-Host "Using CFS for cargo source replacement" + ${env:CARGO_SOURCE_crates-io_REPLACE_WITH} = $null + $env:CARGO_REGISTRIES_CRATESIO_INDEX = $null + + if ($UseCFSAuth) { + if ($null -eq (Get-Command 'az' -ErrorAction Ignore)) { + throw "Azure CLI not found" + } + + if ($null -ne (Get-Command az -ErrorAction Ignore)) { + Write-Host "Getting token" + $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Warning "Failed to get access token, use 'az login' first, or use '-useCratesIO' to use crates.io. Proceeding with anonymous access." + } else { + $header = "Bearer $accessToken" + $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header + $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token' + $env:CARGO_REGISTRIES_POWERSHELL_INDEX = "sparse+https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell~force-auth/Cargo/index/" + } + } + else { + Write-Warning "Azure CLI not found, proceeding with anonymous access." + } + } + } else { + # this will override the config.toml + Write-Host "Setting CARGO_SOURCE_crates-io_REPLACE_WITH to 'crates-io'" + ${env:CARGO_SOURCE_crates-io_REPLACE_WITH} = 'CRATESIO' + $env:CARGO_REGISTRIES_CRATESIO_INDEX = 'sparse+https://index.crates.io/' + } + ## Test if Rust is installed - if (!(Get-Command 'cargo' -ErrorAction Ignore)) { + if (!$usingADO -and !(Get-Command 'cargo' -ErrorAction Ignore)) { Write-Verbose -Verbose "Rust not found, installing..." if (!$IsWindows) { curl https://sh.rustup.rs -sSf | sh -s -- -y @@ -157,15 +200,77 @@ if ($null -ne $packageType) { $env:PATH += ";$env:USERPROFILE\.cargo\bin" Remove-Item temp:/rustup-init.exe -ErrorAction Ignore } + if ($LASTEXITCODE -ne 0) { + throw "Failed to install Rust" + } } - else { + elseif (!$usingADO) { Write-Verbose -Verbose "Rust found, updating..." & $rustup update } - $BuildToolsPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC" + if ($IsWindows) { + $BuildToolsPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC" + } - & $rustup default stable + if (!$usingADO) { + & $rustup default stable + } + + if ($Clippy) { + Write-Verbose -Verbose "Installing clippy..." + if ($UseCFS) { + cargo install clippy --config .cargo/config.toml + } else { + if ($architecture -ne 'current') { + write-verbose -verbose "Installing clippy for $architecture" + rustup component add clippy --target $architecture + } else { + write-verbose -verbose "Installing clippy for current architecture" + rustup component add clippy + } + } + if ($LASTEXITCODE -ne 0) { + throw "Failed to install clippy" + } + } + + ## Test if Node is installed + ## Skipping upgrade as users may have a specific version they want to use + if (!(Get-Command 'node' -ErrorAction Ignore)) { + Write-Verbose -Verbose "Node.js not found, installing..." + if (!$IsWindows) { + if (Get-Command 'brew' -ErrorAction Ignore) { + brew install node@24 + } else { + Write-Warning "Homebrew not found, please install Node.js manually" + } + } + else { + if (Get-Command 'winget' -ErrorAction Ignore) { + Write-Verbose -Verbose "Using winget to install Node.js" + winget install OpenJS.NodeJS --accept-source-agreements --accept-package-agreements --source winget --silent + } else { + Write-Warning "winget not found, please install Node.js manually" + } + } + if ($LASTEXITCODE -ne 0) { + throw "Failed to install Node.js" + } + } + + ## Test if tree-sitter is installed + if ($null -eq (Get-Command tree-sitter -ErrorAction Ignore)) { + Write-Verbose -Verbose "tree-sitter not found, installing..." + if ($UseCFS) { + cargo install tree-sitter-cli --config .cargo/config.toml + } else { + cargo install tree-sitter-cli + } + if ($LASTEXITCODE -ne 0) { + throw "Failed to install tree-sitter-cli" + } + } } if (!$SkipBuild -and !$SkipLinkCheck -and $IsWindows -and !(Get-Command 'link.exe' -ErrorAction Ignore)) { @@ -213,7 +318,7 @@ else { } if (!$SkipBuild) { - if ($architecture -ne 'Current') { + if ($architecture -ne 'Current' -and !$usingADO) { & $rustup target add --toolchain $channel $architecture } @@ -222,39 +327,6 @@ if (!$SkipBuild) { } New-Item -ItemType Directory $target -ErrorAction Ignore > $null - if ($UseCratesIO) { - # this will override the config.toml - Write-Host "Setting CARGO_SOURCE_crates-io_REPLACE_WITH to 'crates-io'" - ${env:CARGO_SOURCE_crates-io_REPLACE_WITH} = 'CRATESIO' - $env:CARGO_REGISTRIES_CRATESIO_INDEX = 'sparse+https://index.crates.io/' - } else { - Write-Host "Using CFS for cargo source replacement" - ${env:CARGO_SOURCE_crates-io_REPLACE_WITH} = $null - $env:CARGO_REGISTRIES_CRATESIO_INDEX = $null - - if ($UseCFSAuth -or $null -ne $env:TF_BUILD) { - if ($null -eq (Get-Command 'az' -ErrorAction Ignore)) { - throw "Azure CLI not found" - } - - if ($null -ne (Get-Command az -ErrorAction Ignore)) { - Write-Host "Getting token" - $accessToken = az account get-access-token --query accessToken --resource 499b84ac-1321-427f-aa17-267ca6975798 -o tsv - if ($LASTEXITCODE -ne 0) { - Write-Warning "Failed to get access token, use 'az login' first, or use '-useCratesIO' to use crates.io. Proceeding with anonymous access." - } else { - $header = "Bearer $accessToken" - $env:CARGO_REGISTRIES_POWERSHELL_INDEX = "sparse+https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell~force-auth/Cargo/index/" - $env:CARGO_REGISTRIES_POWERSHELL_TOKEN = $header - $env:CARGO_REGISTRIES_POWERSHELL_CREDENTIAL_PROVIDER = 'cargo:token' - } - } - else { - Write-Warning "Azure CLI not found, proceeding with anonymous access." - } - } - } - # make sure dependencies are built first so clippy runs correctly $windows_projects = @("pal", "registry", "reboot_pending", "wmi-adapter", "configurations/windows", 'extensions/appx') $macOS_projects = @("resources/brew") @@ -262,6 +334,7 @@ if (!$SkipBuild) { # projects are in dependency order $projects = @( + ".", "tree-sitter-dscexpression", "security_context_lib", "dsc_lib", @@ -273,12 +346,11 @@ if (!$SkipBuild) { "runcommandonset", "tools/dsctest", "tools/test_group_resource", - "y2j", - "." + "y2j" ) $pedantic_unclean_projects = @() - $clippy_unclean_projects = @("tree-sitter-dscexpression") - $skip_test_projects_on_windows = @("tree-sitter-dscexpression") + $clippy_unclean_projects = @("grammars/tree-sitter-dscexpression", "grammars/tree-sitter-ssh-server-config") + $skip_test_projects_on_windows = @("grammars/tree-sitter-dscexpression", "grammars/tree-sitter-ssh-server-config") if ($IsWindows) { $projects += $windows_projects @@ -295,65 +367,81 @@ if (!$SkipBuild) { $failed = $false foreach ($project in $projects) { ## Build format_json - Write-Host -ForegroundColor Cyan "Building $project ... for $architecture" + Write-Host -ForegroundColor Cyan "Building '$project' for $architecture" try { Push-Location "$PSScriptRoot/$project" -ErrorAction Stop + Write-Verbose -Verbose "Current directory is $(Get-Location)" - if ($project -eq 'tree-sitter-dscexpression') { + # check if the project is either tree-sitter-dscexpression or tree-sitter-ssh-server-config + if (($project -match 'tree-sitter-dscexpression$') -or ($project -match 'tree-sitter-ssh-server-config$')) { if ($UpdateLockFile) { cargo generate-lockfile } else { if ($Audit) { if ($null -eq (Get-Command cargo-audit -ErrorAction Ignore)) { - cargo install cargo-audit --features=fix + if ($UseCFS) { + cargo install cargo-audit --features=fix --config .cargo/config.toml + } else { + cargo install cargo-audit --features=fix + } } cargo audit fix } + Write-Verbose -Verbose "Running build.ps1 for $project" ./build.ps1 } } - if (Test-Path "./Cargo.toml") + if ((Test-Path "./Cargo.toml")) { - if ($Clippy) { + $isRepoRoot = $pwd.Path -eq $PSScriptRoot + if ($Clippy -and -not $isRepoRoot) { if ($clippy_unclean_projects -contains $project) { Write-Verbose -Verbose "Skipping clippy for $project" } elseif ($pedantic_unclean_projects -contains $project) { Write-Verbose -Verbose "Running clippy for $project" - cargo clippy @flags -- -Dwarnings + cargo clippy @flags -- -Dwarnings --no-deps } else { Write-Verbose -Verbose "Running clippy with pedantic for $project" - cargo clippy @flags --% -- -Dwarnings -Dclippy::pedantic + cargo clippy @flags --% -- -Dwarnings -Dclippy::pedantic --no-deps } } else { - if ($UpdateLockFile) { + if ($UpdateLockFile -and $isRepoRoot) { cargo generate-lockfile } else { if ($Audit) { if ($null -eq (Get-Command cargo-audit -ErrorAction Ignore)) { - cargo install cargo-audit --features=fix + if ($UseCFS) { + cargo install cargo-audit --features=fix --config .cargo/config.toml + } else { + cargo install cargo-audit --features=fix + } } cargo audit fix } - if ($Clean) { + if ($Clean -and $isRepoRoot) { cargo clean } - cargo build @flags + if (-not $isRepoRoot) { + Write-Verbose -Verbose "Building $project" + cargo build @flags + } } } } - if ($LASTEXITCODE -ne 0) { + if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) { + Write-Error "Last exit code is $LASTEXITCODE, build failed for '$project'" $failed = $true break } @@ -386,10 +474,11 @@ if (!$SkipBuild) { } if ($IsWindows) { - Copy-Item "*.dsc.resource.json" $target -Force -ErrorAction Ignore + Copy-Item "*.dsc.resource.*","*.dsc.manifests.*" $target -Force -ErrorAction Ignore } else { # don't copy WindowsPowerShell resource manifest - Copy-Item "*.dsc.resource.json" $target -Exclude 'windowspowershell.dsc.resource.json' -Force -ErrorAction Ignore + $exclude = @('windowspowershell.dsc.resource.json') + Copy-Item "*.dsc.resource.*","*.dsc.manifests.*" $target -Exclude $exclude -Force -ErrorAction Ignore } # be sure that the files that should be executable are executable @@ -401,7 +490,10 @@ if (!$SkipBuild) { } } } - + } catch { + Write-Error "Failed to build $project : $($_ | Out-String)" + $failed = $true + break } finally { Pop-Location } @@ -440,15 +532,13 @@ if (!$Clippy -and !$SkipBuild) { $env:PATH = [string]::Join([System.IO.Path]::PathSeparator, $env:PATH.Split([System.IO.Path]::PathSeparator, [StringSplitOptions]::RemoveEmptyEntries)) if (!$found) { - Write-Host -ForegroundCOlor Yellow "Adding $target to `$env:PATH" + Write-Host -ForegroundColor Yellow "Adding $target to `$env:PATH" $env:PATH = $target + [System.IO.Path]::PathSeparator + $env:PATH } } if ($Test) { $failed = $false - - $usingADO = ($null -ne $env:TF_BUILD) $repository = 'PSGallery' if ($usingADO) { @@ -474,6 +564,10 @@ if ($Test) { } foreach ($project in $projects) { + # Skip repository root, otherwise it tests all workspace members, failing on not-Windows + if ($project -eq '.') { + continue + } if ($IsWindows -and $skip_test_projects_on_windows -contains $project) { Write-Verbose -Verbose "Skipping test for $project on Windows" continue @@ -522,7 +616,7 @@ if ($Test) { (Get-Module -Name Pester -ListAvailable).Path } - Invoke-Pester -ErrorAction Stop + Invoke-Pester -Output Detailed -ErrorAction Stop } function Find-MakeAppx() {