From 4466803e1fefcb04e0c2257a81ea57423b073b5b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 08:49:58 -0700 Subject: [PATCH] [main] Update dependencies from dotnet/arcade (#4281) * Update dependencies from https://github.com/dotnet/arcade build 20240620.6 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 9.0.0-beta.24317.3 -> To Version 9.0.0-beta.24320.6 * Update dependencies from https://github.com/dotnet/arcade build 20240621.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 9.0.0-beta.24317.3 -> To Version 9.0.0-beta.24321.4 * Update dependencies from https://github.com/dotnet/arcade build 20240621.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 9.0.0-beta.24317.3 -> To Version 9.0.0-beta.24321.4 * Update dependencies from https://github.com/dotnet/arcade build 20240621.4 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk From Version 9.0.0-beta.24317.3 -> To Version 9.0.0-beta.24321.4 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 8 +- eng/common/SetupNugetSources.ps1 | 143 ++++++++++-------- eng/common/SetupNugetSources.sh | 2 +- .../job/publish-build-assets.yml | 9 +- .../post-build/common-variables.yml | 2 - .../core-templates/post-build/post-build.yml | 8 +- .../post-build/setup-maestro-vars.yml | 30 ++-- .../steps/component-governance.yml | 4 +- .../steps/enable-internal-sources.yml | 46 +++--- .../core-templates/steps/source-build.yml | 10 +- eng/common/cross/arm/sources.list.bionic | 11 -- eng/common/cross/arm/sources.list.jessie | 3 - eng/common/cross/arm/sources.list.xenial | 11 -- eng/common/cross/arm/sources.list.zesty | 11 -- eng/common/cross/arm64/sources.list.bionic | 11 -- eng/common/cross/arm64/sources.list.buster | 11 -- eng/common/cross/arm64/sources.list.stretch | 12 -- eng/common/cross/arm64/sources.list.xenial | 11 -- eng/common/cross/arm64/sources.list.zesty | 11 -- eng/common/cross/armel/sources.list.jessie | 3 - eng/common/cross/armv6/sources.list.bookworm | 2 - eng/common/cross/armv6/sources.list.buster | 2 - eng/common/cross/build-rootfs.sh | 41 +++-- eng/common/cross/ppc64le/sources.list.bionic | 11 -- eng/common/cross/riscv64/sources.list.noble | 11 -- eng/common/cross/riscv64/sources.list.sid | 1 - eng/common/cross/s390x/sources.list.bionic | 11 -- eng/common/cross/x86/sources.list.bionic | 11 -- eng/common/cross/x86/sources.list.xenial | 11 -- eng/common/post-build/publish-using-darc.ps1 | 16 +- eng/common/template-guidance.md | 7 +- global.json | 4 +- 32 files changed, 191 insertions(+), 294 deletions(-) delete mode 100644 eng/common/cross/arm/sources.list.bionic delete mode 100644 eng/common/cross/arm/sources.list.jessie delete mode 100644 eng/common/cross/arm/sources.list.xenial delete mode 100644 eng/common/cross/arm/sources.list.zesty delete mode 100644 eng/common/cross/arm64/sources.list.bionic delete mode 100644 eng/common/cross/arm64/sources.list.buster delete mode 100644 eng/common/cross/arm64/sources.list.stretch delete mode 100644 eng/common/cross/arm64/sources.list.xenial delete mode 100644 eng/common/cross/arm64/sources.list.zesty delete mode 100644 eng/common/cross/armel/sources.list.jessie delete mode 100644 eng/common/cross/armv6/sources.list.bookworm delete mode 100644 eng/common/cross/armv6/sources.list.buster delete mode 100644 eng/common/cross/ppc64le/sources.list.bionic delete mode 100644 eng/common/cross/riscv64/sources.list.noble delete mode 100644 eng/common/cross/riscv64/sources.list.sid delete mode 100644 eng/common/cross/s390x/sources.list.bionic delete mode 100644 eng/common/cross/x86/sources.list.bionic delete mode 100644 eng/common/cross/x86/sources.list.xenial diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 81b688f497..4dd48c32e0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,13 +19,13 @@ - + https://github.com/dotnet/arcade - 579b9d3c2a51de22be7685f0bd624bf83265c901 + 3aba80fecac252e1cdaffcebc0a37a24a960228b - + https://github.com/dotnet/arcade - 579b9d3c2a51de22be7685f0bd624bf83265c901 + 3aba80fecac252e1cdaffcebc0a37a24a960228b diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index c07f6a5260..2b0a5c9e66 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,32 +1,31 @@ # This script adds internal feeds required to build commits that depend on internal package sources. For instance, # dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables # disabled internal Maestro (darc-int*) feeds. -# -# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. This credential -# is added via the standard environment variable VSS_NUGET_EXTERNAL_FEED_ENDPOINTS. See -# https://github.com/microsoft/artifacts-credprovider/tree/v1.1.1?tab=readme-ov-file#environment-variables for more details +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # # See example call for this script below. # # - task: PowerShell@2 -# displayName: Setup Internal Feeds +# displayName: Setup Private Feeds Credentials # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -# - task: NuGetAuthenticate@1 -# +# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token +# env: +# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# # Note that the NuGetAuthenticate task should be called after SetupNugetSources. # This ensures that: # - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) -# - The credential provider is installed +# - The credential provider is installed. # # This logic is also abstracted into enable-internal-sources.yml. [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - [string]$Password + $Password ) $ErrorActionPreference = "Stop" @@ -35,23 +34,12 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 -$feedEndpoints = $null - -# If a credential is provided, ensure that we don't overwrite the current set of -# credentials that may have been provided by a previous call to the credential provider. -if ($Password -and $null -ne $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS) { - $feedEndpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS | ConvertFrom-Json -} elseif ($Password) { - $feedEndpoints = @{ endpointCredentials = @() } -} - # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") - if ($null -eq $packageSource) + if ($packageSource -eq $null) { - Write-Host "`tAdding package source" $SourceName $packageSource = $doc.CreateElement("add") $packageSource.SetAttribute("key", $SourceName) $packageSource.SetAttribute("value", $SourceEndPoint) @@ -61,33 +49,63 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) { Write-Host "Package source $SourceName already present." } - if ($pwd) { - $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $SourceEndPoint -pwd $pwd - } + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } -# Add a new feed endpoint credential -function AddCredential([array]$endpointCredentials, $source, $pwd) { - $endpointCredentials += @{ - endpoint = $source; - password = $pwd +# Add a credential node for the specified source +function AddCredential($creds, $source, $username, $pwd) { + # If no cred supplied, don't do anything. + if (!$pwd) { + return; } - return $endpointCredentials + + # Looks for credential configuration for the given SourceName. Create it if none is found. + $sourceElement = $creds.SelectSingleNode($Source) + if ($sourceElement -eq $null) + { + $sourceElement = $doc.CreateElement($Source) + $creds.AppendChild($sourceElement) | Out-Null + } + + # Add the node to the credential if none is found. + $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']") + if ($usernameElement -eq $null) + { + $usernameElement = $doc.CreateElement("add") + $usernameElement.SetAttribute("key", "Username") + $sourceElement.AppendChild($usernameElement) | Out-Null + } + $usernameElement.SetAttribute("value", $Username) + + # Add the to the credential if none is found. + # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs. + # -> https://github.com/NuGet/Home/issues/5526 + $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']") + if ($passwordElement -eq $null) + { + $passwordElement = $doc.CreateElement("add") + $passwordElement.SetAttribute("key", "ClearTextPassword") + $sourceElement.AppendChild($passwordElement) | Out-Null + } + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroInternalFeedCredentials($Sources, $pwd) { - $maestroInternalSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { + $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($PackageSource in $maestroInternalSources) { - Write-Host "`tAdding credential for Maestro's feed:" $PackageSource.Key - $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $PackageSource.value -pwd $pwd + Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." + + ForEach ($PackageSource in $maestroPrivateSources) { + Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } -function EnableInternalPackageSources($DisabledPackageSources) { - $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroInternalSources) { - Write-Host "`tEnsuring internal source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" +function EnablePrivatePackageSources($DisabledPackageSources) { + $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroPrivateSources) { + Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) } @@ -105,27 +123,38 @@ $doc.Load($filename) # Get reference to or create one if none exist already $sources = $doc.DocumentElement.SelectSingleNode("packageSources") -if ($null -eq $sources) { +if ($sources -eq $null) { $sources = $doc.CreateElement("packageSources") $doc.DocumentElement.AppendChild($sources) | Out-Null } +$creds = $null +if ($Password) { + # Looks for a node. Create it if none is found. + $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") + if ($creds -eq $null) { + $creds = $doc.CreateElement("packageSourceCredentials") + $doc.DocumentElement.AppendChild($creds) | Out-Null + } +} + # Check for disabledPackageSources; we'll enable any darc-int ones we find there $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") -if ($null -ne $disabledSources) { +if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnableInternalPackageSources -DisabledPackageSources $disabledSources + EnablePrivatePackageSources -DisabledPackageSources $disabledSources } -if ($Password) { - InsertMaestroInternalFeedCredentials -Sources $sources -pwd $Password -} +$userName = "dn-bot" + +# Insert credential nodes for Maestro's private feeds +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($null -ne $dotnet31Source) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json" -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json" -pwd $Password +if ($dotnet31Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -133,18 +162,10 @@ $dotnetVersions = @('5','6','7','8') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") - if ($dotnetSource) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedprefix-internal/nuget/v3/index.json" -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v3/index.json" -pwd $Password + if ($dotnetSource -ne $null) { + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } $doc.Save($filename) - -# If any credentials were added or altered, update the VSS_NUGET_EXTERNAL_FEED_ENDPOINTS environment variable -if ($null -ne $feedEndpoints) { - # ci is set to true so vso logging commands will be used. - $ci = $true - Write-PipelineSetVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $($feedEndpoints | ConvertTo-Json) -IsMultiJobVariable $false - Write-PipelineSetVariable -Name 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' -Value "False" -IsMultiJobVariable $false -} \ No newline at end of file diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 16c1e29ea3..b493479a1d 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This script adds internal feeds required to build commits that depend on intenral package sources. For instance, +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, # dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables # disabled internal Maestro (darc-int*) feeds. # diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index 8fe9299542..2cf8e1853d 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -87,13 +87,15 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -160,7 +162,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml index b9ede10bf0..d5627a994a 100644 --- a/eng/common/core-templates/post-build/common-variables.yml +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -8,8 +8,6 @@ variables: # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - name: MaestroApiVersion value: "2020-02-20" diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index fb15c40c03..20924366b8 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -300,14 +300,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index 8d56b57267..f7602980db 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -15,19 +15,20 @@ steps: artifactName: ReleaseConfigs checkDownloadedFiles: true - - task: PowerShell@2 + - task: AzureCLI@2 name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - pwsh: true - script: | + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | try { if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 + $Channels = $Content | Select -Index 1 $IsStableBuild = $Content | Select -Index 2 $AzureDevOpsProject = $Env:System_TeamProject @@ -35,15 +36,16 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId + $BarId = ${{ parameters.BARBuildId }} $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" $Channels = "[$Channels]" @@ -69,6 +71,4 @@ steps: exit 1 } env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml index df449a34c1..b8815892a5 100644 --- a/eng/common/core-templates/steps/component-governance.yml +++ b/eng/common/core-templates/steps/component-governance.yml @@ -2,7 +2,8 @@ parameters: disableComponentGovernance: false componentGovernanceIgnoreDirectories: '' is1ESPipeline: false - + displayName: 'Component Detection' + steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" @@ -10,5 +11,6 @@ steps: - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 continueOnError: true + displayName: ${{ parameters.displayName }} inputs: ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml index 80deddafb1..4a06b52908 100644 --- a/eng/common/core-templates/steps/enable-internal-sources.yml +++ b/eng/common/core-templates/steps/enable-internal-sources.yml @@ -6,30 +6,40 @@ parameters: - name: is1ESPipeline type: boolean default: false +# Legacy parameters to allow for PAT usage +- name: legacyCredential + type: string + default: '' steps: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. - # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that - # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token. - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if ne(parameters.legacyCredential, '') }}: - task: PowerShell@2 displayName: Setup Internal Feeds inputs: filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config - - task: NuGetAuthenticate@1 + # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. + # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that + # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token. - ${{ else }}: - - template: /eng/common/templates/steps/get-federated-access-token.yml - parameters: - federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }} - outputVariableName: 'dnceng-artifacts-feeds-read-access-token' - - task: PowerShell@2 - displayName: Setup Internal Feeds - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) - # This is required in certain scenarios to install the ADO credential provider. - # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others - # (e.g. dotnet msbuild). - - task: NuGetAuthenticate@1 + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config + - ${{ else }}: + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }} + outputVariableName: 'dnceng-artifacts-feeds-read-access-token' + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + # This is required in certain scenarios to install the ADO credential provider. + # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others + # (e.g. dotnet msbuild). + - task: NuGetAuthenticate@1 diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index 16c778d92c..2915d29bb7 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -121,7 +121,9 @@ steps: # a nupkg cache of input packages (a local feed). # This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' # in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + displayName: Component Detection (Exclude upstream cache) + is1ESPipeline: ${{ parameters.is1ESPipeline }} + componentGovernanceIgnoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + disableComponentGovernance: ${{ eq(variables['System.TeamProject'], 'public') }} diff --git a/eng/common/cross/arm/sources.list.bionic b/eng/common/cross/arm/sources.list.bionic deleted file mode 100644 index 2109557409..0000000000 --- a/eng/common/cross/arm/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jessie b/eng/common/cross/arm/sources.list.jessie deleted file mode 100644 index 4d142ac9b1..0000000000 --- a/eng/common/cross/arm/sources.list.jessie +++ /dev/null @@ -1,3 +0,0 @@ -# Debian (sid) # UNSTABLE -deb http://ftp.debian.org/debian/ sid main contrib non-free -deb-src http://ftp.debian.org/debian/ sid main contrib non-free diff --git a/eng/common/cross/arm/sources.list.xenial b/eng/common/cross/arm/sources.list.xenial deleted file mode 100644 index 56fbb36a59..0000000000 --- a/eng/common/cross/arm/sources.list.xenial +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.zesty b/eng/common/cross/arm/sources.list.zesty deleted file mode 100644 index ea2c14a787..0000000000 --- a/eng/common/cross/arm/sources.list.zesty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.bionic b/eng/common/cross/arm64/sources.list.bionic deleted file mode 100644 index 2109557409..0000000000 --- a/eng/common/cross/arm64/sources.list.bionic +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.buster b/eng/common/cross/arm64/sources.list.buster deleted file mode 100644 index 7194ac64a9..0000000000 --- a/eng/common/cross/arm64/sources.list.buster +++ /dev/null @@ -1,11 +0,0 @@ -deb http://deb.debian.org/debian buster main -deb-src http://deb.debian.org/debian buster main - -deb http://deb.debian.org/debian-security/ buster/updates main -deb-src http://deb.debian.org/debian-security/ buster/updates main - -deb http://deb.debian.org/debian buster-updates main -deb-src http://deb.debian.org/debian buster-updates main - -deb http://deb.debian.org/debian buster-backports main contrib non-free -deb-src http://deb.debian.org/debian buster-backports main contrib non-free diff --git a/eng/common/cross/arm64/sources.list.stretch b/eng/common/cross/arm64/sources.list.stretch deleted file mode 100644 index 0e12157743..0000000000 --- a/eng/common/cross/arm64/sources.list.stretch +++ /dev/null @@ -1,12 +0,0 @@ -deb http://deb.debian.org/debian stretch main -deb-src http://deb.debian.org/debian stretch main - -deb http://deb.debian.org/debian-security/ stretch/updates main -deb-src http://deb.debian.org/debian-security/ stretch/updates main - -deb http://deb.debian.org/debian stretch-updates main -deb-src http://deb.debian.org/debian stretch-updates main - -deb http://deb.debian.org/debian stretch-backports main contrib non-free -deb-src http://deb.debian.org/debian stretch-backports main contrib non-free - diff --git a/eng/common/cross/arm64/sources.list.xenial b/eng/common/cross/arm64/sources.list.xenial deleted file mode 100644 index 56fbb36a59..0000000000 --- a/eng/common/cross/arm64/sources.list.xenial +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.zesty b/eng/common/cross/arm64/sources.list.zesty deleted file mode 100644 index ea2c14a787..0000000000 --- a/eng/common/cross/arm64/sources.list.zesty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse diff --git a/eng/common/cross/armel/sources.list.jessie b/eng/common/cross/armel/sources.list.jessie deleted file mode 100644 index 3d9c3059d8..0000000000 --- a/eng/common/cross/armel/sources.list.jessie +++ /dev/null @@ -1,3 +0,0 @@ -# Debian (jessie) # Stable -deb http://ftp.debian.org/debian/ jessie main contrib non-free -deb-src http://ftp.debian.org/debian/ jessie main contrib non-free diff --git a/eng/common/cross/armv6/sources.list.bookworm b/eng/common/cross/armv6/sources.list.bookworm deleted file mode 100644 index 1016113526..0000000000 --- a/eng/common/cross/armv6/sources.list.bookworm +++ /dev/null @@ -1,2 +0,0 @@ -deb http://raspbian.raspberrypi.org/raspbian/ bookworm main contrib non-free rpi -deb-src http://raspbian.raspberrypi.org/raspbian/ bookworm main contrib non-free rpi diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster deleted file mode 100644 index f27fc4fb34..0000000000 --- a/eng/common/cross/armv6/sources.list.buster +++ /dev/null @@ -1,2 +0,0 @@ -deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi -deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8bb233ba25..eb1a908046 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -30,7 +30,8 @@ __IllumosArch=arm7 __HaikuArch=arm __QEMUArch=arm __UbuntuArch=armhf -__UbuntuRepo="http://ports.ubuntu.com/" +__UbuntuRepo= +__UbuntuSuites="updates security backports" __LLDB_Package="liblldb-3.9-dev" __SkipUnmount=0 @@ -129,9 +130,9 @@ __AlpineKeys=' 616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ== ' __Keyring= +__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg" __SkipSigCheck=0 __UseMirror=0 -__UseDeb822Format=0 __UnprocessedBuildArgs= while :; do @@ -163,6 +164,7 @@ while :; do __UbuntuArch=armel __UbuntuRepo="http://ftp.debian.org/debian/" __CodeName=jessie + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" ;; armv6) __BuildArch=armv6 @@ -170,10 +172,12 @@ while :; do __QEMUArch=arm __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" __CodeName=buster + __KeyringFile="/usr/share/keyrings/raspbian-archive-keyring.gpg" __LLDB_Package="liblldb-6.0-dev" + __UbuntuSuites= - if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg" + if [[ -e "$__KeyringFile" ]]; then + __Keyring="--keyring $__KeyringFile" fi ;; riscv64) @@ -184,10 +188,6 @@ while :; do __UbuntuArch=riscv64 __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" unset __LLDB_Package - - if [[ -e "/usr/share/keyrings/debian-archive-keyring.gpg" ]]; then - __Keyring="--keyring /usr/share/keyrings/debian-archive-keyring.gpg --include=debian-archive-keyring" - fi ;; ppc64le) __BuildArch=ppc64le @@ -292,10 +292,13 @@ while :; do if [[ "$__CodeName" != "jessie" ]]; then __CodeName=noble fi - __UseDeb822Format=1 + if [[ -n "$__LLDB_Package" ]]; then + __LLDB_Package="liblldb-18-dev" + fi ;; jessie) # Debian 8 __CodeName=jessie + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -304,6 +307,7 @@ while :; do stretch) # Debian 9 __CodeName=stretch __LLDB_Package="liblldb-6.0-dev" + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -312,6 +316,7 @@ while :; do buster) # Debian 10 __CodeName=buster __LLDB_Package="liblldb-6.0-dev" + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -319,6 +324,7 @@ while :; do ;; bullseye) # Debian 11 __CodeName=bullseye + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -326,6 +332,7 @@ while :; do ;; bookworm) # Debian 12 __CodeName=bookworm + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -333,6 +340,7 @@ while :; do ;; sid) # Debian sid __CodeName=sid + __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg" if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" @@ -442,6 +450,10 @@ fi __UbuntuPackages+=" ${__LLDB_Package:-}" +if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ports.ubuntu.com/" +fi + if [[ -n "$__LLVM_MajorVersion" ]]; then __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" fi @@ -740,9 +752,16 @@ elif [[ -n "$__CodeName" ]]; then # shellcheck disable=SC2086 echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + mkdir -p "$__RootfsDir/etc/apt/sources.list.d/" - grep -q "Types:" "$__CrossDir/$__BuildArch/sources.list.$__CodeName" && filename="$__CodeName.sources" || filename="$__CodeName.list" - cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list.d/$filename" + cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" <