diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 86a60ca986..d243ff7970 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -3,9 +3,9 @@
-
+
https://github.com/dotnet/arcade
- c4bbc67763bf0c5a868862df874079380e647d61
+ 7d1967403f5b12406763c666f41e3358bb542ced
diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd
index 56c2f25ac2..ac1f72bf94 100644
--- a/eng/common/CIBuild.cmd
+++ b/eng/common/CIBuild.cmd
@@ -1,2 +1,2 @@
@echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
\ No newline at end of file
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
diff --git a/eng/common/build.ps1 b/eng/common/build.ps1
index 438f9920c4..6b3be1916f 100644
--- a/eng/common/build.ps1
+++ b/eng/common/build.ps1
@@ -7,6 +7,7 @@ Param(
[string] $msbuildEngine = $null,
[bool] $warnAsError = $true,
[bool] $nodeReuse = $true,
+ [switch] $buildCheck = $false,
[switch][Alias('r')]$restore,
[switch] $deployDeps,
[switch][Alias('b')]$build,
@@ -71,6 +72,8 @@ function Print-Usage() {
Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
+ Write-Host " -nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
+ Write-Host " -buildCheck Sets /check msbuild parameter"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -97,6 +100,7 @@ function Build {
$bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
$platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
+ $check = if ($buildCheck) { '/check' } else { '' }
if ($projects) {
# Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@@ -113,6 +117,7 @@ function Build {
MSBuild $toolsetBuildProj `
$bl `
$platformArg `
+ $check `
/p:Configuration=$configuration `
/p:RepoRoot=$RepoRoot `
/p:Restore=$restore `
@@ -127,6 +132,7 @@ function Build {
/p:PerformanceTest=$performanceTest `
/p:Sign=$sign `
/p:Publish=$publish `
+ /p:RestoreStaticGraphEnableBinaryLogger=$binaryLog `
@properties
}
diff --git a/eng/common/build.sh b/eng/common/build.sh
index ac1ee8620c..36fba82a37 100755
--- a/eng/common/build.sh
+++ b/eng/common/build.sh
@@ -42,6 +42,7 @@ usage()
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')"
echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')"
+ echo " --buildCheck Sets /check msbuild parameter"
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
@@ -76,6 +77,7 @@ clean=false
warn_as_error=true
node_reuse=true
+build_check=false
binary_log=false
exclude_ci_binary_log=false
pipelines_log=false
@@ -173,6 +175,9 @@ while [[ $# > 0 ]]; do
node_reuse=$2
shift
;;
+ -buildcheck)
+ build_check=true
+ ;;
-runtimesourcefeed)
runtime_source_feed=$2
shift
@@ -224,14 +229,19 @@ function Build {
bl="/bl:\"$log_dir/Build.binlog\""
fi
+ local check=""
+ if [[ "$build_check" == true ]]; then
+ check="/check"
+ fi
+
MSBuild $_InitializeToolset \
$bl \
+ $check \
/p:Configuration=$configuration \
/p:RepoRoot="$repo_root" \
/p:Restore=$restore \
/p:Build=$build \
/p:DotNetBuildRepo=$product_build \
- /p:ArcadeBuildFromSource=$source_build \
/p:DotNetBuildSourceOnly=$source_build \
/p:Rebuild=$rebuild \
/p:Test=$test \
@@ -240,6 +250,7 @@ function Build {
/p:PerformanceTest=$performance_test \
/p:Sign=$sign \
/p:Publish=$publish \
+ /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \
$properties
ExitWithExitCode 0
diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh
index 1a02c0dec8..66e3b0ac61 100755
--- a/eng/common/cibuild.sh
+++ b/eng/common/cibuild.sh
@@ -13,4 +13,4 @@ while [[ -h $source ]]; do
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
\ No newline at end of file
+. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@
diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml
index ba53ebfbd5..4dfb3d8683 100644
--- a/eng/common/core-templates/job/job.yml
+++ b/eng/common/core-templates/job/job.yml
@@ -19,10 +19,10 @@ parameters:
# publishing defaults
artifacts: ''
enableMicrobuild: false
+ enableMicrobuildForMacAndLinux: false
enablePublishBuildArtifacts: false
enablePublishBuildAssets: false
enablePublishTestResults: false
- enablePublishUsingPipelines: false
enableBuildRetry: false
mergeTestResults: false
testRunTitle: ''
@@ -127,18 +127,11 @@ jobs:
- ${{ preStep }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- - task: MicroBuildSigningPlugin@4
- displayName: Install MicroBuild plugin
- inputs:
- signType: $(_SignType)
- zipSources: false
- feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
- env:
- TeamName: $(_TeamName)
- MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)'
+ - template: /eng/common/core-templates/steps/install-microbuild.yml
+ parameters:
+ enableMicrobuild: ${{ parameters.enableMicrobuild }}
+ enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
continueOnError: ${{ parameters.continueOnError }}
- condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
- ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
- task: NuGetAuthenticate@1
@@ -167,14 +160,12 @@ jobs:
- ${{ each step in parameters.componentGovernanceSteps }}:
- ${{ step }}
- - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - task: MicroBuildCleanup@1
- displayName: Execute Microbuild cleanup tasks
- condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - template: /eng/common/core-templates/steps/cleanup-microbuild.yml
+ parameters:
+ enableMicrobuild: ${{ parameters.enableMicrobuild }}
+ enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
continueOnError: ${{ parameters.continueOnError }}
- env:
- TeamName: $(_TeamName)
# Publish test results
- ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml
index 3d3356e319..5da3c2a2a2 100644
--- a/eng/common/core-templates/job/publish-build-assets.yml
+++ b/eng/common/core-templates/job/publish-build-assets.yml
@@ -20,9 +20,6 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
- # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
- publishUsingPipelines: false
-
# Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
publishAssetsImmediately: false
@@ -76,12 +73,11 @@ jobs:
fetchDepth: 3
clean: true
- - task: DownloadBuildArtifacts@0
- displayName: Download artifact
+ - task: DownloadPipelineArtifact@2
+ displayName: Download Asset Manifests
inputs:
artifactName: AssetManifests
- downloadPath: '$(Build.StagingDirectory)/Download'
- checkDownloadedFiles: true
+ targetPath: '$(Build.StagingDirectory)/AssetManifests'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
@@ -95,9 +91,8 @@ jobs:
scriptLocation: scriptPath
scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
- /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
+ /p:ManifestsPath='$(Build.StagingDirectory)/AssetManifests'
/p:MaestroApiEndpoint=https://maestro.dot.net
- /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:OfficialBuildId=$(Build.BuildNumber)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml
index c4713c8b6e..05f7ad6ef0 100644
--- a/eng/common/core-templates/job/source-build.yml
+++ b/eng/common/core-templates/job/source-build.yml
@@ -12,9 +12,10 @@ parameters:
# The name of the job. This is included in the job ID.
# targetRID: ''
# The name of the target RID to use, instead of the one auto-detected by Arcade.
- # nonPortable: false
+ # portableBuild: false
# Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
- # linux-x64), and compiling against distro-provided packages rather than portable ones.
+ # linux-x64), and compiling against distro-provided packages rather than portable ones. The
+ # default is portable mode.
# skipPublishValidation: false
# Disables publishing validation. By default, a check is performed to ensure no packages are
# published by source-build.
diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml
index 205fb5b3a3..30530359a5 100644
--- a/eng/common/core-templates/job/source-index-stage1.yml
+++ b/eng/common/core-templates/job/source-index-stage1.yml
@@ -1,8 +1,5 @@
parameters:
runAsPublic: false
- sourceIndexUploadPackageVersion: 2.0.0-20240522.1
- sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
- sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
@@ -16,12 +13,6 @@ jobs:
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- - name: SourceIndexUploadPackageVersion
- value: ${{ parameters.sourceIndexUploadPackageVersion }}
- - name: SourceIndexProcessBinlogPackageVersion
- value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }}
- - name: SourceIndexPackageSource
- value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- template: /eng/common/core-templates/variables/pool-providers.yml
@@ -34,12 +25,10 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $(DncEngPublicBuildPool)
- image: 1es-windows-2022-open
- os: windows
+ image: windows.vs2022.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $(DncEngInternalBuildPool)
- image: 1es-windows-2022
- os: windows
+ image: windows.vs2022.amd64
steps:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
@@ -47,35 +36,9 @@ jobs:
- ${{ each preStep in parameters.preSteps }}:
- ${{ preStep }}
-
- - task: UseDotNet@2
- displayName: Use .NET 8 SDK
- inputs:
- packageType: sdk
- version: 8.0.x
- installationPath: $(Agent.TempDirectory)/dotnet
- workingDirectory: $(Agent.TempDirectory)
-
- - script: |
- $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
- $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
- displayName: Download Tools
- # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
- workingDirectory: $(Agent.TempDirectory)
-
- script: ${{ parameters.sourceIndexBuildCommand }}
displayName: Build Repository
- - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
- displayName: Process Binlog into indexable sln
-
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - task: AzureCLI@2
- displayName: Log in to Azure and upload stage1 artifacts to source index
- inputs:
- azureSubscription: 'SourceDotNet Stage1 Publish'
- addSpnToEnvironment: true
- scriptType: 'ps'
- scriptLocation: 'inlineScript'
- inlineScript: |
- $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
+ - template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
+ parameters:
+ binLogPath: ${{ parameters.binLogPath }}
\ No newline at end of file
diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml
index f2144252cc..693b00b370 100644
--- a/eng/common/core-templates/jobs/codeql-build.yml
+++ b/eng/common/core-templates/jobs/codeql-build.yml
@@ -15,7 +15,6 @@ jobs:
enablePublishBuildArtifacts: false
enablePublishTestResults: false
enablePublishBuildAssets: false
- enablePublishUsingPipelines: false
enableTelemetry: true
variables:
diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml
index ea69be4341..3d2deaa4a0 100644
--- a/eng/common/core-templates/jobs/jobs.yml
+++ b/eng/common/core-templates/jobs/jobs.yml
@@ -5,9 +5,6 @@ parameters:
# Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false
- # Optional: Enable publishing using release pipelines
- enablePublishUsingPipelines: false
-
# Optional: Enable running the source-build jobs to build repo from source
enableSourceBuild: false
@@ -112,7 +109,6 @@ jobs:
- Source_Build_Complete
runAsPublic: ${{ parameters.runAsPublic }}
- publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml
index 454fd75c7a..a8c0bd3b92 100644
--- a/eng/common/core-templates/post-build/post-build.yml
+++ b/eng/common/core-templates/post-build/post-build.yml
@@ -44,6 +44,11 @@ parameters:
displayName: Publish installers and checksums
type: boolean
default: true
+
+ - name: requireDefaultChannels
+ displayName: Fail the build if there are no default channel(s) registrations for the current build
+ type: boolean
+ default: false
- name: SDLValidationParameters
type: object
@@ -312,5 +317,6 @@ stages:
-PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
-AzdoToken '$(System.AccessToken)'
-WaitPublishingFinish true
+ -RequireDefaultChannels ${{ parameters.requireDefaultChannels }}
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/core-templates/steps/cleanup-microbuild.yml b/eng/common/core-templates/steps/cleanup-microbuild.yml
new file mode 100644
index 0000000000..c0fdcd3379
--- /dev/null
+++ b/eng/common/core-templates/steps/cleanup-microbuild.yml
@@ -0,0 +1,28 @@
+parameters:
+ # Enable cleanup tasks for MicroBuild
+ enableMicrobuild: false
+ # Enable cleanup tasks for MicroBuild on Mac and Linux
+ # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+ enableMicrobuildForMacAndLinux: false
+ continueOnError: false
+
+steps:
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - task: MicroBuildCleanup@1
+ displayName: Execute Microbuild cleanup tasks
+ condition: and(
+ always(),
+ or(
+ and(
+ eq(variables['Agent.Os'], 'Windows_NT'),
+ in(variables['_SignType'], 'real', 'test')
+ ),
+ and(
+ ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+ ne(variables['Agent.Os'], 'Windows_NT'),
+ eq(variables['_SignType'], 'real')
+ )
+ ))
+ continueOnError: ${{ parameters.continueOnError }}
+ env:
+ TeamName: $(_TeamName)
diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml
index d938b60e1b..44a9636cdf 100644
--- a/eng/common/core-templates/steps/generate-sbom.yml
+++ b/eng/common/core-templates/steps/generate-sbom.yml
@@ -5,7 +5,7 @@
# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
parameters:
- PackageVersion: 9.0.0
+ PackageVersion: 10.0.0
BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
PackageName: '.NET'
ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
@@ -38,7 +38,7 @@ steps:
PackageName: ${{ parameters.packageName }}
BuildDropPath: ${{ parameters.buildDropPath }}
PackageVersion: ${{ parameters.packageVersion }}
- ManifestDirPath: ${{ parameters.manifestDirPath }}
+ ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME)
${{ if ne(parameters.IgnoreDirectories, '') }}:
AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
diff --git a/eng/common/core-templates/steps/get-delegation-sas.yml b/eng/common/core-templates/steps/get-delegation-sas.yml
index 9db5617ea7..d2901470a7 100644
--- a/eng/common/core-templates/steps/get-delegation-sas.yml
+++ b/eng/common/core-templates/steps/get-delegation-sas.yml
@@ -31,16 +31,7 @@ steps:
# Calculate the expiration of the SAS token and convert to UTC
$expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
- # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
- # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
- $sas = ""
- do {
- $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
- if ($LASTEXITCODE -ne 0) {
- Write-Error "Failed to generate SAS token."
- exit 1
- }
- } while($sas.IndexOf('/') -ne -1)
+ $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
if ($LASTEXITCODE -ne 0) {
Write-Error "Failed to generate SAS token."
diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml
new file mode 100644
index 0000000000..dba506e74c
--- /dev/null
+++ b/eng/common/core-templates/steps/install-microbuild.yml
@@ -0,0 +1,99 @@
+parameters:
+ # Enable install tasks for MicroBuild
+ enableMicrobuild: false
+ # Enable install tasks for MicroBuild on Mac and Linux
+ # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
+ enableMicrobuildForMacAndLinux: false
+ # Location of the MicroBuild output folder
+ microBuildOutputFolder: '$(Build.SourcesDirectory)'
+ continueOnError: false
+
+steps:
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+ - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
+ # Install Python 3.12.x on when Python > 3.12.x is installed - https://github.com/dotnet/source-build/issues/4802
+ - script: |
+ version=$(python3 --version | awk '{print $2}')
+ major=$(echo $version | cut -d. -f1)
+ minor=$(echo $version | cut -d. -f2)
+
+ installPython=false
+ if [ "$major" -gt 3 ] || { [ "$major" -eq 3 ] && [ "$minor" -gt 12 ]; }; then
+ installPython=true
+ fi
+
+ echo "Python version: $version."
+ echo "Install Python 3.12.x: $installPython."
+ echo "##vso[task.setvariable variable=installPython;isOutput=true]$installPython"
+ name: InstallPython
+ displayName: 'Determine Python installation'
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '3.12.x'
+ displayName: 'Use Python 3.12.x'
+ condition: and(succeeded(), eq(variables['InstallPython.installPython'], 'true'), ne(variables['Agent.Os'], 'Windows_NT'))
+
+ # Needed to download the MicroBuild plugin nupkgs on Mac and Linux when nuget.exe is unavailable
+ - task: UseDotNet@2
+ displayName: Install .NET 8.0 SDK for MicroBuild Plugin
+ inputs:
+ packageType: sdk
+ version: 8.0.x
+ installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet
+ workingDirectory: ${{ parameters.microBuildOutputFolder }}
+ condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
+ - task: MicroBuildSigningPlugin@4
+ displayName: Install MicroBuild plugin
+ inputs:
+ signType: $(_SignType)
+ zipSources: false
+ feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+ ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
+ azureSubscription: 'MicroBuild Signing Task (DevDiv)'
+ useEsrpCli: true
+ env:
+ TeamName: $(_TeamName)
+ MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(
+ succeeded(),
+ or(
+ and(
+ eq(variables['Agent.Os'], 'Windows_NT'),
+ in(variables['_SignType'], 'real', 'test')
+ ),
+ and(
+ ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }},
+ ne(variables['Agent.Os'], 'Windows_NT'),
+ eq(variables['_SignType'], 'real')
+ )
+ ))
+
+ # Workaround for ESRP CLI on Linux - https://github.com/dotnet/source-build/issues/4964
+ - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
+ - task: UseDotNet@2
+ displayName: Install .NET 9.0 SDK for ESRP CLI Workaround
+ inputs:
+ packageType: sdk
+ version: 9.0.x
+ installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet
+ workingDirectory: ${{ parameters.microBuildOutputFolder }}
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
+
+ - task: PowerShell@2
+ displayName: Workaround for ESRP CLI on Linux
+ inputs:
+ targetType: 'inline'
+ script: |
+ Write-Host "Copying Linux Path"
+ $MBSIGN_APPFOLDER = '$(MBSIGN_APPFOLDER)'
+ $MBSIGN_APPFOLDER = $MBSIGN_APPFOLDER -replace '/build', ''
+ $MBSIGN_APPFOLDER = $MBSIGN_APPFOLDER + '/1.1.1032' + '/build'
+ $MBSIGN_APPFOLDER | Write-Host
+ $SignConfigPath = $MBSIGN_APPFOLDER + '/signconfig.xml'
+ Copy-Item -Path "$(MBSIGN_APPFOLDER)/signconfig.xml" -Destination $SignConfigPath -Force
+ condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml
index 80788c5231..de24d0087c 100644
--- a/eng/common/core-templates/steps/publish-logs.yml
+++ b/eng/common/core-templates/steps/publish-logs.yml
@@ -34,7 +34,9 @@ steps:
'$(akams-client-id)'
'$(microsoft-symbol-server-pat)'
'$(symweb-symbol-server-pat)'
+ '$(dnceng-symbol-server-pat)'
'$(dn-bot-all-orgs-build-rw-code-rw)'
+ '$(System.AccessToken)'
${{parameters.CustomSensitiveDataList}}
continueOnError: true
condition: always()
@@ -45,6 +47,7 @@ steps:
SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs'
Contents: '**'
TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs'
+ condition: always()
- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
parameters:
diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml
index 2915d29bb7..2341706b06 100644
--- a/eng/common/core-templates/steps/source-build.yml
+++ b/eng/common/core-templates/steps/source-build.yml
@@ -37,7 +37,7 @@ steps:
# in the default public locations.
internalRuntimeDownloadArgs=
if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then
- internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
+ internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://ci.dot.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)'
fi
buildConfig=Release
@@ -76,6 +76,11 @@ steps:
assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml
fi
+ portableBuildArgs=
+ if [ '${{ parameters.platform.portableBuild }}' != '' ]; then
+ portableBuildArgs='/p:PortableBuild=${{ parameters.platform.portableBuild }}'
+ fi
+
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \
--restore --build --pack $publishArgs -bl \
@@ -85,8 +90,7 @@ steps:
$targetRidArgs \
$runtimeOsArgs \
$baseOsArgs \
- /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
- /p:ArcadeBuildFromSource=true \
+ $portableBuildArgs \
/p:DotNetBuildSourceOnly=true \
/p:DotNetBuildRepo=true \
/p:AssetManifestFileName=$assetManifestFileName
diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml
new file mode 100644
index 0000000000..473a22c471
--- /dev/null
+++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml
@@ -0,0 +1,35 @@
+parameters:
+ sourceIndexUploadPackageVersion: 2.0.0-20240522.1
+ sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1
+ sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
+ binlogPath: artifacts/log/Debug/Build.binlog
+
+steps:
+- task: UseDotNet@2
+ displayName: "Source Index: Use .NET 8 SDK"
+ inputs:
+ packageType: sdk
+ version: 8.0.x
+ installationPath: $(Agent.TempDirectory)/dotnet
+ workingDirectory: $(Agent.TempDirectory)
+
+- script: |
+ $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
+ $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
+ displayName: "Source Index: Download netsourceindex Tools"
+ # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
+ workingDirectory: $(Agent.TempDirectory)
+
+- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
+ displayName: "Source Index: Process Binlog into indexable sln"
+
+- ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: AzureCLI@2
+ displayName: "Source Index: Upload Source Index stage1 artifacts to Azure"
+ inputs:
+ azureSubscription: 'SourceDotNet Stage1 Publish'
+ addSpnToEnvironment: true
+ scriptType: 'ps'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch
deleted file mode 100644
index 2d26156193..0000000000
--- a/eng/common/cross/armel/armel.jessie.patch
+++ /dev/null
@@ -1,43 +0,0 @@
-diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
---- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
-+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
-@@ -69,10 +69,10 @@
- #endif
- #ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
-- return __sync_val_compare_and_swap_2(addr, old, _new);
-+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
- #endif
- case 4:
-- return __sync_val_compare_and_swap_4(addr, old, _new);
-+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_val_compare_and_swap_8(addr, old, _new);
-@@ -109,7 +109,7 @@
- return;
- #endif
- case 4:
-- __sync_and_and_fetch_4(addr, val);
-+ __sync_and_and_fetch_4((uint32_t*) addr, val);
- return;
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
-@@ -148,7 +148,7 @@
- return;
- #endif
- case 4:
-- __sync_or_and_fetch_4(addr, val);
-+ __sync_or_and_fetch_4((uint32_t*) addr, val);
- return;
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
-@@ -187,7 +187,7 @@
- return __sync_add_and_fetch_2(addr, val);
- #endif
- case 4:
-- return __sync_add_and_fetch_4(addr, val);
-+ return __sync_add_and_fetch_4((uint32_t*) addr, val);
- #if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_add_and_fetch_8(addr, val);
diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh
index 7e9ba2b75e..fbd8d80848 100644
--- a/eng/common/cross/build-android-rootfs.sh
+++ b/eng/common/cross/build-android-rootfs.sh
@@ -6,10 +6,11 @@ usage()
{
echo "Creates a toolchain and sysroot used for cross-compiling for Android."
echo
- echo "Usage: $0 [BuildArch] [ApiLevel]"
+ echo "Usage: $0 [BuildArch] [ApiLevel] [--ndk NDKVersion]"
echo
echo "BuildArch is the target architecture of Android. Currently only arm64 is supported."
echo "ApiLevel is the target Android API level. API levels usually match to Android releases. See https://source.android.com/source/build-numbers.html"
+ echo "NDKVersion is the version of Android NDK. The default is r21. See https://developer.android.com/ndk/downloads/revision_history"
echo
echo "By default, the toolchain and sysroot will be generated in cross/android-rootfs/toolchain/[BuildArch]. You can change this behavior"
echo "by setting the TOOLCHAIN_DIR environment variable"
@@ -25,10 +26,15 @@ __BuildArch=arm64
__AndroidArch=aarch64
__AndroidToolchain=aarch64-linux-android
-for i in "$@"
- do
- lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
- case $lowerI in
+while :; do
+ if [[ "$#" -le 0 ]]; then
+ break
+ fi
+
+ i=$1
+
+ lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
-?|-h|--help)
usage
exit 1
@@ -43,6 +49,10 @@ for i in "$@"
__AndroidArch=arm
__AndroidToolchain=arm-linux-androideabi
;;
+ --ndk)
+ shift
+ __NDK_Version=$1
+ ;;
*[0-9])
__ApiLevel=$i
;;
@@ -50,8 +60,17 @@ for i in "$@"
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $i"
;;
esac
+ shift
done
+if [[ "$__NDK_Version" == "r21" ]] || [[ "$__NDK_Version" == "r22" ]]; then
+ __NDK_File_Arch_Spec=-x86_64
+ __SysRoot=sysroot
+else
+ __NDK_File_Arch_Spec=
+ __SysRoot=toolchains/llvm/prebuilt/linux-x86_64/sysroot
+fi
+
# Obtain the location of the bash script to figure out where the root of the repo is.
__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -78,6 +97,7 @@ fi
echo "Target API level: $__ApiLevel"
echo "Target architecture: $__BuildArch"
+echo "NDK version: $__NDK_Version"
echo "NDK location: $__NDK_Dir"
echo "Target Toolchain location: $__ToolchainDir"
@@ -85,8 +105,8 @@ echo "Target Toolchain location: $__ToolchainDir"
if [ ! -d $__NDK_Dir ]; then
echo Downloading the NDK into $__NDK_Dir
mkdir -p $__NDK_Dir
- wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip
- unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir
+ wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux$__NDK_File_Arch_Spec.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux.zip
+ unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux.zip -d $__CrossDir
fi
if [ ! -d $__lldb_Dir ]; then
@@ -116,16 +136,11 @@ for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/
fi
done
-cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/"
+cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/$__SysRoot/usr/"
# Generate platform file for build.sh script to assign to __DistroRid
echo "Generating platform file..."
-echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform
-
-echo "Now to build coreclr, libraries and installers; run:"
-echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
- --subsetCategory coreclr
-echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
- --subsetCategory libraries
-echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
- --subsetCategory installer
+echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/$__SysRoot/android_platform
+
+echo "Now to build coreclr, libraries and host; run:"
+echo ROOTFS_DIR=$(realpath $__ToolchainDir/$__SysRoot) ./build.sh clr+libs+host --cross --arch $__BuildArch
diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh
index 4b5e8d7166..d6f005b5da 100644
--- a/eng/common/cross/build-rootfs.sh
+++ b/eng/common/cross/build-rootfs.sh
@@ -5,7 +5,7 @@ set -e
usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]"
- echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
+ echo "BuildArch can be: arm(default), arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine"
echo " for alpine can be specified with version: alpineX.YY or alpineedge"
echo " for FreeBSD can be: freebsd13, freebsd14"
@@ -15,6 +15,7 @@ usage()
echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--skipsigcheck - optional, will skip package signature checks (allowing untrusted packages)."
+ echo "--skipemulation - optional, will skip qemu and debootstrap requirement when building environment for debian based systems."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
echo "--jobs N - optional, restrict to N jobs."
exit 1
@@ -52,28 +53,27 @@ __UbuntuPackages+=" symlinks"
__UbuntuPackages+=" libicu-dev"
__UbuntuPackages+=" liblttng-ust-dev"
__UbuntuPackages+=" libunwind8-dev"
-__UbuntuPackages+=" libnuma-dev"
__AlpinePackages+=" gettext-dev"
__AlpinePackages+=" icu-dev"
__AlpinePackages+=" libunwind-dev"
__AlpinePackages+=" lttng-ust-dev"
__AlpinePackages+=" compiler-rt"
-__AlpinePackages+=" numactl-dev"
# runtime libraries' dependencies
__UbuntuPackages+=" libcurl4-openssl-dev"
__UbuntuPackages+=" libkrb5-dev"
__UbuntuPackages+=" libssl-dev"
__UbuntuPackages+=" zlib1g-dev"
+__UbuntuPackages+=" libbrotli-dev"
__AlpinePackages+=" curl-dev"
__AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
-__FreeBSDBase="13.3-RELEASE"
-__FreeBSDPkg="1.17.0"
+__FreeBSDBase="13.4-RELEASE"
+__FreeBSDPkg="1.21.3"
__FreeBSDABI="13"
__FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu"
@@ -91,18 +91,18 @@ __HaikuPackages="gcc_syslibs"
__HaikuPackages+=" gcc_syslibs_devel"
__HaikuPackages+=" gmp"
__HaikuPackages+=" gmp_devel"
-__HaikuPackages+=" icu66"
-__HaikuPackages+=" icu66_devel"
+__HaikuPackages+=" icu[0-9]+"
+__HaikuPackages+=" icu[0-9]*_devel"
__HaikuPackages+=" krb5"
__HaikuPackages+=" krb5_devel"
__HaikuPackages+=" libiconv"
__HaikuPackages+=" libiconv_devel"
-__HaikuPackages+=" llvm12_libunwind"
-__HaikuPackages+=" llvm12_libunwind_devel"
+__HaikuPackages+=" llvm[0-9]*_libunwind"
+__HaikuPackages+=" llvm[0-9]*_libunwind_devel"
__HaikuPackages+=" mpfr"
__HaikuPackages+=" mpfr_devel"
-__HaikuPackages+=" openssl"
-__HaikuPackages+=" openssl_devel"
+__HaikuPackages+=" openssl3"
+__HaikuPackages+=" openssl3_devel"
__HaikuPackages+=" zlib"
__HaikuPackages+=" zlib_devel"
@@ -128,10 +128,12 @@ __AlpineKeys='
616adfeb:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU\npNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X\n0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6\nADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ\nraQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy\nKLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj\n/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H\nKuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs\nmFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1\n/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq\nXIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4\nVORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ==
616ae350:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0\nXkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ\npiM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46\nxLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP\nODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM\nEyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr\nDzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+\nf53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF\nHwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk\nOk9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6\n9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT\nJZDr++FjKrnnijbyNF8b98UCAwEAAQ==
616db30d:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0\nlIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm\na6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw\ntO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C\nUS/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP\nhP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv\nLSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0\nx9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF\nwmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG\nLPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV\nGZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C\n8CM1S15HxV78s9dFntEqIokCAwEAAQ==
+66ba20fe:MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtfB12w4ZgqsXWZDfUAV/\n6Y4aHUKIu3q4SXrNZ7CXF9nXoAVYrS7NAxJdAodsY3vPCN0g5O8DFXR+390LdOuQ\n+HsGKCc1k5tX5ZXld37EZNTNSbR0k+NKhd9h6X3u6wqPOx7SIKxwAQR8qeeFq4pP\nrt9GAGlxtuYgzIIcKJPwE0dZlcBCg+GnptCUZXp/38BP1eYC+xTXSL6Muq1etYfg\nodXdb7Yl+2h1IHuOwo5rjgY5kpY7GcAs8AjGk3lDD/av60OTYccknH0NCVSmPoXK\nvrxDBOn0LQRNBLcAfnTKgHrzy0Q5h4TNkkyTgxkoQw5ObDk9nnabTxql732yy9BY\ns+hM9+dSFO1HKeVXreYSA2n1ndF18YAvAumzgyqzB7I4pMHXq1kC/8bONMJxwSkS\nYm6CoXKyavp7RqGMyeVpRC7tV+blkrrUml0BwNkxE+XnwDRB3xDV6hqgWe0XrifD\nYTfvd9ScZQP83ip0r4IKlq4GMv/R5shcCRJSkSZ6QSGshH40JYSoiwJf5FHbj9ND\n7do0UAqebWo4yNx63j/wb2ULorW3AClv0BCFSdPsIrCStiGdpgJDBR2P2NZOCob3\nG9uMj+wJD6JJg2nWqNJxkANXX37Qf8plgzssrhrgOvB0fjjS7GYhfkfmZTJ0wPOw\nA8+KzFseBh4UFGgue78KwgkCAwEAAQ==
'
__Keyring=
__KeyringFile="/usr/share/keyrings/ubuntu-archive-keyring.gpg"
__SkipSigCheck=0
+__SkipEmulation=0
__UseMirror=0
__UnprocessedBuildArgs=
@@ -162,9 +164,13 @@ while :; do
armel)
__BuildArch=armel
__UbuntuArch=armel
- __UbuntuRepo="http://ftp.debian.org/debian/"
- __CodeName=jessie
+ __UbuntuRepo="http://archive.debian.org/debian/"
+ __CodeName=buster
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+ __LLDB_Package="liblldb-6.0-dev"
+ __UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
+ __UbuntuPackages="${__UbuntuPackages// libomp5/}"
+ __UbuntuSuites=
;;
armv6)
__BuildArch=armv6
@@ -180,6 +186,18 @@ while :; do
__Keyring="--keyring $__KeyringFile"
fi
;;
+ loongarch64)
+ __BuildArch=loongarch64
+ __AlpineArch=loongarch64
+ __QEMUArch=loongarch64
+ __UbuntuArch=loong64
+ __UbuntuSuites=unreleased
+ __LLDB_Package="liblldb-19-dev"
+
+ if [[ "$__CodeName" == "sid" ]]; then
+ __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
+ fi
+ ;;
riscv64)
__BuildArch=riscv64
__AlpineArch=riscv64
@@ -264,46 +282,23 @@ while :; do
;;
xenial) # Ubuntu 16.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=xenial
- fi
- ;;
- zesty) # Ubuntu 17.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=zesty
- fi
+ __CodeName=xenial
;;
bionic) # Ubuntu 18.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=bionic
- fi
+ __CodeName=bionic
;;
focal) # Ubuntu 20.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=focal
- fi
+ __CodeName=focal
;;
jammy) # Ubuntu 22.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=jammy
- fi
+ __CodeName=jammy
;;
noble) # Ubuntu 24.04
- if [[ "$__CodeName" != "jessie" ]]; then
- __CodeName=noble
- fi
+ __CodeName=noble
if [[ -n "$__LLDB_Package" ]]; then
__LLDB_Package="liblldb-18-dev"
fi
;;
- jessie) # Debian 8
- __CodeName=jessie
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
-
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
- fi
- ;;
stretch) # Debian 9
__CodeName=stretch
__LLDB_Package="liblldb-6.0-dev"
@@ -319,7 +314,7 @@ while :; do
__KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ __UbuntuRepo="http://archive.debian.org/debian/"
fi
;;
bullseye) # Debian 11
@@ -340,10 +335,28 @@ while :; do
;;
sid) # Debian sid
__CodeName=sid
- __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+ __UbuntuSuites=
- if [[ -z "$__UbuntuRepo" ]]; then
- __UbuntuRepo="http://ftp.debian.org/debian/"
+ # Debian-Ports architectures need different values
+ case "$__UbuntuArch" in
+ amd64|arm64|armel|armhf|i386|mips64el|ppc64el|riscv64|s390x)
+ __KeyringFile="/usr/share/keyrings/debian-archive-keyring.gpg"
+
+ if [[ -z "$__UbuntuRepo" ]]; then
+ __UbuntuRepo="http://ftp.debian.org/debian/"
+ fi
+ ;;
+ *)
+ __KeyringFile="/usr/share/keyrings/debian-ports-archive-keyring.gpg"
+
+ if [[ -z "$__UbuntuRepo" ]]; then
+ __UbuntuRepo="http://ftp.ports.debian.org/debian-ports/"
+ fi
+ ;;
+ esac
+
+ if [[ -e "$__KeyringFile" ]]; then
+ __Keyring="--keyring $__KeyringFile"
fi
;;
tizen)
@@ -370,7 +383,7 @@ while :; do
;;
freebsd14)
__CodeName=freebsd
- __FreeBSDBase="14.0-RELEASE"
+ __FreeBSDBase="14.2-RELEASE"
__FreeBSDABI="14"
__SkipUnmount=1
;;
@@ -388,6 +401,9 @@ while :; do
--skipsigcheck)
__SkipSigCheck=1
;;
+ --skipemulation)
+ __SkipEmulation=1
+ ;;
--rootfsdir|-rootfsdir)
shift
__RootfsDir="$1"
@@ -420,16 +436,15 @@ case "$__AlpineVersion" in
elif [[ "$__AlpineArch" == "x86" ]]; then
__AlpineVersion=3.17 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm15-libs"
- elif [[ "$__AlpineArch" == "riscv64" ]]; then
+ elif [[ "$__AlpineArch" == "riscv64" || "$__AlpineArch" == "loongarch64" ]]; then
+ __AlpineVersion=3.21 # minimum version that supports lldb-dev
+ __AlpinePackages+=" llvm19-libs"
+ elif [[ -n "$__AlpineMajorVersion" ]]; then
+ # use whichever alpine version is provided and select the latest toolchain libs
__AlpineLlvmLibsLookup=1
- __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive)
else
__AlpineVersion=3.13 # 3.13 to maximize compatibility
__AlpinePackages+=" llvm10-libs"
-
- if [[ "$__AlpineArch" == "armv7" ]]; then
- __AlpinePackages="${__AlpinePackages//numactl-dev/}"
- fi
fi
esac
@@ -439,15 +454,6 @@ if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then
__AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}"
fi
-if [[ "$__BuildArch" == "armel" ]]; then
- __LLDB_Package="lldb-3.5-dev"
-fi
-
-if [[ "$__CodeName" == "xenial" && "$__UbuntuArch" == "armhf" ]]; then
- # libnuma-dev is not available on armhf for xenial
- __UbuntuPackages="${__UbuntuPackages//libnuma-dev/}"
-fi
-
__UbuntuPackages+=" ${__LLDB_Package:-}"
if [[ -z "$__UbuntuRepo" ]]; then
@@ -496,7 +502,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
arch="$(uname -m)"
ensureDownloadTool
-
+
if [[ "$__hasWget" == 1 ]]; then
wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
else
@@ -512,11 +518,6 @@ if [[ "$__CodeName" == "alpine" ]]; then
echo "$__ApkToolsSHA512SUM $__ApkToolsDir/apk.static" | sha512sum -c
chmod +x "$__ApkToolsDir/apk.static"
- if [[ -f "/usr/bin/qemu-$__QEMUArch-static" ]]; then
- mkdir -p "$__RootfsDir"/usr/bin
- cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
- fi
-
if [[ "$__AlpineVersion" == "edge" ]]; then
version=edge
else
@@ -536,6 +537,10 @@ if [[ "$__CodeName" == "alpine" ]]; then
__ApkSignatureArg="--keys-dir $__ApkKeysDir"
fi
+ if [[ "$__SkipEmulation" == "1" ]]; then
+ __NoEmulationArg="--no-scripts"
+ fi
+
# initialize DB
# shellcheck disable=SC2086
"$__ApkToolsDir/apk.static" \
@@ -557,7 +562,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
"$__ApkToolsDir/apk.static" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
- -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
+ -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" $__NoEmulationArg \
add $__AlpinePackages
rm -r "$__ApkToolsDir"
@@ -573,7 +578,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
fi
echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
- echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
+ echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
mkdir -p "$__RootfsDir"/tmp
# get and build package manager
if [[ "$__hasWget" == 1 ]]; then
@@ -681,7 +686,7 @@ elif [[ "$__CodeName" == "haiku" ]]; then
ensureDownloadTool
- echo "Downloading Haiku package tool"
+ echo "Downloading Haiku package tools"
git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
if [[ "$__hasWget" == 1 ]]; then
wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
@@ -691,34 +696,42 @@ elif [[ "$__CodeName" == "haiku" ]]; then
unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
- DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
- HpkgBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+ HaikuBaseUrl="https://eu.hpkg.haiku-os.org/haiku/master/$__HaikuArch/current"
+ HaikuPortsBaseUrl="https://eu.hpkg.haiku-os.org/haikuports/master/$__HaikuArch/current"
+
+ echo "Downloading HaikuPorts package repository index..."
+ if [[ "$__hasWget" == 1 ]]; then
+ wget -P "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
+ else
+ curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuPortsBaseUrl/repo"
+ fi
- # Download Haiku packages
echo "Downloading Haiku packages"
read -ra array <<<"$__HaikuPackages"
for package in "${array[@]}"; do
echo "Downloading $package..."
- # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
- # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
+ hpkgFilename="$(LD_LIBRARY_PATH="$__RootfsDir/tmp/bin" "$__RootfsDir/tmp/bin/package_repo" list -f "$__RootfsDir/tmp/download/repo" |
+ grep -E "${package}-" | sort -V | tail -n 1 | xargs)"
+ if [ -z "$hpkgFilename" ]; then
+ >&2 echo "ERROR: package $package missing."
+ exit 1
+ fi
+ echo "Resolved filename: $hpkgFilename..."
+ hpkgDownloadUrl="$HaikuPortsBaseUrl/packages/$hpkgFilename"
if [[ "$__hasWget" == 1 ]]; then
- hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
- --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
else
- hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
- --header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
fi
done
for package in haiku haiku_devel; do
echo "Downloading $package..."
if [[ "$__hasWget" == 1 ]]; then
- hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
- wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+ hpkgVersion="$(wget -qO- "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+ wget -P "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
else
- hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
- curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
+ hpkgVersion="$(curl -sSL "$HaikuBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
+ curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HaikuBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
fi
done
@@ -744,25 +757,67 @@ elif [[ "$__CodeName" == "haiku" ]]; then
popd
rm -rf "$__RootfsDir/tmp"
elif [[ -n "$__CodeName" ]]; then
+ __Suites="$__CodeName $(for suite in $__UbuntuSuites; do echo -n "$__CodeName-$suite "; done)"
+
+ if [[ "$__SkipEmulation" == "1" ]]; then
+ if [[ -z "$AR" ]]; then
+ if command -v ar &>/dev/null; then
+ AR="$(command -v ar)"
+ elif command -v llvm-ar &>/dev/null; then
+ AR="$(command -v llvm-ar)"
+ else
+ echo "Unable to find ar or llvm-ar on PATH, add them to PATH or set AR environment variable pointing to the available AR tool"
+ exit 1
+ fi
+ fi
+
+ PYTHON=${PYTHON_EXECUTABLE:-python3}
+
+ # shellcheck disable=SC2086,SC2046
+ echo running "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
+ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \
+ $__UbuntuPackages
+
+ # shellcheck disable=SC2086,SC2046
+ "$PYTHON" "$__CrossDir/install-debs.py" --arch "$__UbuntuArch" --mirror "$__UbuntuRepo" --rootfsdir "$__RootfsDir" --artool "$AR" \
+ $(for suite in $__Suites; do echo -n "--suite $suite "; done) \
+ $__UbuntuPackages
+ exit 0
+ fi
+
+ __UpdateOptions=
if [[ "$__SkipSigCheck" == "0" ]]; then
__Keyring="$__Keyring --force-check-gpg"
+ else
+ __Keyring=
+ __UpdateOptions="--allow-unauthenticated --allow-insecure-repositories"
fi
# shellcheck disable=SC2086
echo running debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
- debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
+ # shellcheck disable=SC2086
+ if ! debootstrap "--variant=minbase" $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"; then
+ echo "debootstrap failed! dumping debootstrap.log"
+ cat "$__RootfsDir/debootstrap/debootstrap.log"
+ exit 1
+ fi
+
+ rm -rf "$__RootfsDir"/etc/apt/*.{sources,list} "$__RootfsDir"/etc/apt/sources.list.d
mkdir -p "$__RootfsDir/etc/apt/sources.list.d/"
+
+ # shellcheck disable=SC2086
cat > "$__RootfsDir/etc/apt/sources.list.d/$__CodeName.sources" < token2) - (token1 < token2)
+ else:
+ return -1 if isinstance(token1, str) else 1
+
+ return len(tokens1) - len(tokens2)
+
+def compare_debian_versions(version1, version2):
+ """Compare two Debian package versions."""
+ epoch1, upstream1, revision1 = parse_debian_version(version1)
+ epoch2, upstream2, revision2 = parse_debian_version(version2)
+
+ if epoch1 != epoch2:
+ return epoch1 - epoch2
+
+ result = compare_upstream_version(upstream1, upstream2)
+ if result != 0:
+ return result
+
+ return compare_upstream_version(revision1, revision2)
+
+def resolve_dependencies(packages, aliases, desired_packages):
+ """Recursively resolves dependencies for the desired packages."""
+ resolved = []
+ to_process = deque(desired_packages)
+
+ while to_process:
+ current = to_process.popleft()
+ resolved_package = current if current in packages else aliases.get(current, [None])[0]
+
+ if not resolved_package:
+ print(f"Error: Package '{current}' was not found in the available packages.")
+ sys.exit(1)
+
+ if resolved_package not in resolved:
+ resolved.append(resolved_package)
+
+ deps = packages.get(resolved_package, {}).get("Depends", "")
+ if deps:
+ deps = [dep.split(' ')[0] for dep in deps.split(', ') if dep]
+ for dep in deps:
+ if dep not in resolved and dep not in to_process and dep in packages:
+ to_process.append(dep)
+
+ return resolved
+
+def parse_package_index(content):
+ """Parses the Packages.gz file and returns package information."""
+ packages = {}
+ aliases = {}
+ entries = re.split(r'\n\n+', content)
+
+ for entry in entries:
+ fields = dict(re.findall(r'^(\S+): (.+)$', entry, re.MULTILINE))
+ if "Package" in fields:
+ package_name = fields["Package"]
+ version = fields.get("Version")
+ filename = fields.get("Filename")
+ depends = fields.get("Depends")
+ provides = fields.get("Provides", None)
+
+ # Only update if package_name is not in packages or if the new version is higher
+ if package_name not in packages or compare_debian_versions(version, packages[package_name]["Version"]) > 0:
+ packages[package_name] = {
+ "Version": version,
+ "Filename": filename,
+ "Depends": depends
+ }
+
+ # Update aliases if package provides any alternatives
+ if provides:
+ provides_list = [x.strip() for x in provides.split(",")]
+ for alias in provides_list:
+ # Strip version specifiers
+ alias_name = re.sub(r'\s*\(=.*\)', '', alias)
+ if alias_name not in aliases:
+ aliases[alias_name] = []
+ if package_name not in aliases[alias_name]:
+ aliases[alias_name].append(package_name)
+
+ return packages, aliases
+
+def install_packages(mirror, packages_info, aliases, tmp_dir, extract_dir, ar_tool, desired_packages):
+ """Downloads .deb files and extracts them."""
+ resolved_packages = resolve_dependencies(packages_info, aliases, desired_packages)
+ print(f"Resolved packages (including dependencies): {resolved_packages}")
+
+ packages_to_download = {}
+
+ for pkg in resolved_packages:
+ if pkg in packages_info:
+ packages_to_download[pkg] = packages_info[pkg]
+
+ if pkg in aliases:
+ for alias in aliases[pkg]:
+ if alias in packages_info:
+ packages_to_download[alias] = packages_info[alias]
+
+ asyncio.run(download_deb_files_parallel(mirror, packages_to_download, tmp_dir))
+
+ package_to_deb_file_map = {}
+ for pkg in resolved_packages:
+ pkg_info = packages_info.get(pkg)
+ if pkg_info:
+ deb_filename = pkg_info.get("Filename")
+ if deb_filename:
+ deb_file_path = os.path.join(tmp_dir, os.path.basename(deb_filename))
+ package_to_deb_file_map[pkg] = deb_file_path
+
+ for pkg in reversed(resolved_packages):
+ deb_file = package_to_deb_file_map.get(pkg)
+ if deb_file and os.path.exists(deb_file):
+ extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool)
+
+ print("All done!")
+
+def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool):
+ """Extract .deb file contents"""
+
+ os.makedirs(extract_dir, exist_ok=True)
+
+ with tempfile.TemporaryDirectory(dir=tmp_dir) as tmp_subdir:
+ result = subprocess.run(f"{ar_tool} t {os.path.abspath(deb_file)}", cwd=tmp_subdir, check=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+ tar_filename = None
+ for line in result.stdout.decode().splitlines():
+ if line.startswith("data.tar"):
+ tar_filename = line.strip()
+ break
+
+ if not tar_filename:
+ raise FileNotFoundError(f"Could not find 'data.tar.*' in {deb_file}.")
+
+ tar_file_path = os.path.join(tmp_subdir, tar_filename)
+ print(f"Extracting {tar_filename} from {deb_file}..")
+
+ subprocess.run(f"{ar_tool} p {os.path.abspath(deb_file)} {tar_filename} > {tar_file_path}", check=True, shell=True)
+
+ file_extension = os.path.splitext(tar_file_path)[1].lower()
+
+ if file_extension == ".xz":
+ mode = "r:xz"
+ elif file_extension == ".gz":
+ mode = "r:gz"
+ elif file_extension == ".zst":
+ # zstd is not supported by standard library yet
+ decompressed_tar_path = tar_file_path.replace(".zst", "")
+ with open(tar_file_path, "rb") as zst_file, open(decompressed_tar_path, "wb") as decompressed_file:
+ dctx = zstandard.ZstdDecompressor()
+ dctx.copy_stream(zst_file, decompressed_file)
+
+ tar_file_path = decompressed_tar_path
+ mode = "r"
+ else:
+ raise ValueError(f"Unsupported compression format: {file_extension}")
+
+ with tarfile.open(tar_file_path, mode) as tar:
+ tar.extractall(path=extract_dir, filter='fully_trusted')
+
+def finalize_setup(rootfsdir):
+ lib_dir = os.path.join(rootfsdir, 'lib')
+ usr_lib_dir = os.path.join(rootfsdir, 'usr', 'lib')
+
+ if os.path.exists(lib_dir):
+ if os.path.islink(lib_dir):
+ os.remove(lib_dir)
+ else:
+ os.makedirs(usr_lib_dir, exist_ok=True)
+
+ for item in os.listdir(lib_dir):
+ src = os.path.join(lib_dir, item)
+ dest = os.path.join(usr_lib_dir, item)
+
+ if os.path.isdir(src):
+ shutil.copytree(src, dest, dirs_exist_ok=True)
+ else:
+ shutil.copy2(src, dest)
+
+ shutil.rmtree(lib_dir)
+
+ os.symlink(usr_lib_dir, lib_dir)
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Generate rootfs for .NET runtime on Debian-like OS")
+ parser.add_argument("--distro", required=False, help="Distro name (e.g., debian, ubuntu, etc.)")
+ parser.add_argument("--arch", required=True, help="Architecture (e.g., amd64, loong64, etc.)")
+ parser.add_argument("--rootfsdir", required=True, help="Destination directory.")
+ parser.add_argument('--suite', required=True, action='append', help='Specify one or more repository suites to collect index data.')
+ parser.add_argument("--mirror", required=False, help="Mirror (e.g., http://ftp.debian.org/debian-ports etc.)")
+ parser.add_argument("--artool", required=False, default="ar", help="ar tool to extract debs (e.g., ar, llvm-ar etc.)")
+ parser.add_argument("packages", nargs="+", help="List of package names to be installed.")
+
+ args = parser.parse_args()
+
+ if args.mirror is None:
+ if args.distro == "ubuntu":
+ args.mirror = "http://archive.ubuntu.com/ubuntu" if args.arch in ["amd64", "i386"] else "http://ports.ubuntu.com/ubuntu-ports"
+ elif args.distro == "debian":
+ args.mirror = "http://ftp.debian.org/debian-ports"
+ else:
+ raise Exception("Unsupported distro")
+
+ DESIRED_PACKAGES = args.packages + [ # base packages
+ "dpkg",
+ "busybox",
+ "libc-bin",
+ "base-files",
+ "base-passwd",
+ "debianutils"
+ ]
+
+ print(f"Creating rootfs. rootfsdir: {args.rootfsdir}, distro: {args.distro}, arch: {args.arch}, suites: {args.suite}, mirror: {args.mirror}")
+
+ package_index_content = asyncio.run(download_package_index_parallel(args.mirror, args.arch, args.suite))
+
+ packages_info, aliases = parse_package_index(package_index_content)
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ install_packages(args.mirror, packages_info, aliases, tmp_dir, args.rootfsdir, args.artool, DESIRED_PACKAGES)
+
+ finalize_setup(args.rootfsdir)
diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake
index 9a7ecfbd42..0ff85cf036 100644
--- a/eng/common/cross/toolchain.cmake
+++ b/eng/common/cross/toolchain.cmake
@@ -67,6 +67,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "armv6")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif()
+elseif(TARGET_ARCH_NAME STREQUAL "loongarch64")
+ set(CMAKE_SYSTEM_PROCESSOR "loongarch64")
+ if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/loongarch64-alpine-linux-musl)
+ set(TOOLCHAIN "loongarch64-alpine-linux-musl")
+ else()
+ set(TOOLCHAIN "loongarch64-linux-gnu")
+ endif()
elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
set(CMAKE_SYSTEM_PROCESSOR ppc64le)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/powerpc64le-alpine-linux-musl)
@@ -118,7 +125,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu")
endif()
else()
- message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
+ message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, loongarch64, ppc64le, riscv64, s390x, x64 and x86 are supported!")
endif()
if(DEFINED ENV{TOOLCHAIN})
@@ -148,6 +155,25 @@ if(TIZEN)
include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN})
endif()
+function(locate_toolchain_exec exec var)
+ set(TOOLSET_PREFIX ${TOOLCHAIN}-)
+ string(TOUPPER ${exec} EXEC_UPPERCASE)
+ if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
+ set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
+ return()
+ endif()
+
+ find_program(EXEC_LOCATION_${exec}
+ NAMES
+ "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
+ "${TOOLSET_PREFIX}${exec}")
+
+ if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
+ message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
+ endif()
+ set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
+endfunction()
+
if(ANDROID)
if(TARGET_ARCH_NAME STREQUAL "arm")
set(ANDROID_ABI armeabi-v7a)
@@ -178,66 +204,24 @@ elseif(FREEBSD)
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld")
elseif(ILLUMOS)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+ set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
include_directories(SYSTEM ${CROSS_ROOTFS}/include)
- set(TOOLSET_PREFIX ${TOOLCHAIN}-)
- function(locate_toolchain_exec exec var)
- string(TOUPPER ${exec} EXEC_UPPERCASE)
- if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
- set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
- return()
- endif()
-
- find_program(EXEC_LOCATION_${exec}
- NAMES
- "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
- "${TOOLSET_PREFIX}${exec}")
-
- if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
- message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
- endif()
- set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
- endfunction()
-
- set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
-
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
-
- set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
- set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
-
- set(TOOLSET_PREFIX ${TOOLCHAIN}-)
- function(locate_toolchain_exec exec var)
- string(TOUPPER ${exec} EXEC_UPPERCASE)
- if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
- set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
- return()
- endif()
-
- find_program(EXEC_LOCATION_${exec}
- NAMES
- "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
- "${TOOLSET_PREFIX}${exec}")
-
- if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
- message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
- endif()
- set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
- endfunction()
-
set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
+ set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
+ set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
- set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
- set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
-
# let CMake set up the correct search paths
include(Platform/Haiku)
else()
@@ -307,7 +291,7 @@ endif()
# Specify compile options
-if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
+if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|loongarch64|ppc64le|riscv64|s390x|x64|x86)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1
index 3e5c1c74a1..a0c7d792a7 100644
--- a/eng/common/generate-sbom-prep.ps1
+++ b/eng/common/generate-sbom-prep.ps1
@@ -4,18 +4,26 @@ Param(
. $PSScriptRoot\pipeline-logging-functions.ps1
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM"
+$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_'
+$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName
+
+Write-Host "Artifact name before : $ArtifactName"
+Write-Host "Artifact name after : $SafeArtifactName"
+
Write-Host "Creating dir $ManifestDirPath"
+
# create directory for sbom manifest to be placed
-if (!(Test-Path -path $ManifestDirPath))
+if (!(Test-Path -path $SbomGenerationDir))
{
- New-Item -ItemType Directory -path $ManifestDirPath
- Write-Host "Successfully created directory $ManifestDirPath"
+ New-Item -ItemType Directory -path $SbomGenerationDir
+ Write-Host "Successfully created directory $SbomGenerationDir"
}
else{
Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
}
Write-Host "Updating artifact name"
-$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
-Write-Host "Artifact name $artifact_name"
-Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName"
diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh
index d5c76dc827..b8ecca72bb 100755
--- a/eng/common/generate-sbom-prep.sh
+++ b/eng/common/generate-sbom-prep.sh
@@ -14,19 +14,24 @@ done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. $scriptroot/pipeline-logging-functions.sh
+
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
manifest_dir=$1
-if [ ! -d "$manifest_dir" ] ; then
- mkdir -p "$manifest_dir"
- echo "Sbom directory created." $manifest_dir
+# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly
+# with their own overwriting ours. So we create it as a sub directory of the requested manifest path.
+sbom_generation_dir="$manifest_dir/$safe_artifact_name"
+
+if [ ! -d "$sbom_generation_dir" ] ; then
+ mkdir -p "$sbom_generation_dir"
+ echo "Sbom directory created." $sbom_generation_dir
else
Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder."
fi
-artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
echo "Artifact name before : "$artifact_name
-# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
-safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
echo "Artifact name after : "$safe_artifact_name
export ARTIFACT_NAME=$safe_artifact_name
echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj
index 32f79dfb34..feaa6d2081 100644
--- a/eng/common/internal/Tools.csproj
+++ b/eng/common/internal/Tools.csproj
@@ -15,16 +15,6 @@
-
-
-
- https://devdiv.pkgs.visualstudio.com/_packaging/dotnet-core-internal-tooling/nuget/v3/index.json;
-
-
- $(RestoreSources);
- https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json;
-
-
diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh
new file mode 100644
index 0000000000..477a44f335
--- /dev/null
+++ b/eng/common/native/install-dependencies.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+
+set -e
+
+# This is a simple script primarily used for CI to install necessary dependencies
+#
+# Usage:
+#
+# ./install-dependencies.sh
+
+os="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
+
+if [ -z "$os" ]; then
+ . "$(dirname "$0")"/init-os-and-arch.sh
+fi
+
+case "$os" in
+ linux)
+ if [ -e /etc/os-release ]; then
+ . /etc/os-release
+ fi
+
+ if [ "$ID" = "debian" ] || [ "$ID_LIKE" = "debian" ]; then
+ apt update
+
+ apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
+ libssl-dev libkrb5-dev pigz cpio
+
+ localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+ elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then
+ pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)"
+ $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio
+ elif [ "$ID" = "alpine" ]; then
+ apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio
+ else
+ echo "Unsupported distro. distro: $ID"
+ exit 1
+ fi
+ ;;
+
+ osx|maccatalyst|ios|iossimulator|tvos|tvossimulator)
+ echo "Installed xcode version: $(xcode-select -p)"
+
+ export HOMEBREW_NO_INSTALL_CLEANUP=1
+ export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+ # Skip brew update for now, see https://github.com/actions/setup-python/issues/577
+ # brew update --preinstall
+ brew bundle --no-upgrade --file=- < Name of Arcade task (name of a project in SdkTasks directory of the Arcade SDK package)"
+ echo " --restore Restore dependencies"
+ echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Command line arguments not listed above are passed thru to msbuild."
+}
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+ source="$(readlink "$source")"
+ # if $source was a relative symlink, we need to resolve it relative to the path where the
+ # symlink file was located
+ [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+Build() {
+ local target=$1
+ local log_suffix=""
+ [[ "$target" != "Execute" ]] && log_suffix=".$target"
+ local log="$log_dir/$task$log_suffix.binlog"
+ local output_path="$toolset_dir/$task/"
+
+ MSBuild "$taskProject" \
+ /bl:"$log" \
+ /t:"$target" \
+ /p:Configuration="$configuration" \
+ /p:RepoRoot="$repo_root" \
+ /p:BaseIntermediateOutputPath="$output_path" \
+ /v:"$verbosity" \
+ $properties
+}
+
+configuration="Debug"
+verbosity="minimal"
+restore=false
+help=false
+properties=''
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
+ case $lowerI in
+ --task)
+ task=$2
+ shift 2
+ ;;
+ --restore)
+ restore=true
+ shift 1
+ ;;
+ --verbosity)
+ verbosity=$2
+ shift 2
+ ;;
+ --help)
+ help=true
+ shift 1
+ ;;
+ *)
+ properties="$properties $1"
+ shift 1
+ ;;
+ esac
+done
+
+ci=true
+binaryLog=true
+warnAsError=true
+
+if $help; then
+ show_usage
+ exit 0
+fi
+
+. "$scriptroot/tools.sh"
+InitializeToolset
+
+if [[ -z "$task" ]]; then
+ Write-PipelineTelemetryError -Category 'Task' -Name 'MissingTask' -Message "Missing required parameter '-task '"
+ ExitWithExitCode 1
+fi
+
+taskProject=$(GetSdkTaskProject "$task")
+if [[ ! -e "$taskProject" ]]; then
+ Write-PipelineTelemetryError -Category 'Task' -Name 'UnknownTask' -Message "Unknown task: $task"
+ ExitWithExitCode 1
+fi
+
+if $restore; then
+ Build "Restore"
+fi
+
+Build "Execute"
+
+
+ExitWithExitCode 0
diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml
index 605692d2fb..a8a9432874 100644
--- a/eng/common/templates-official/job/job.yml
+++ b/eng/common/templates-official/job/job.yml
@@ -16,6 +16,7 @@ jobs:
parameters:
PackageVersion: ${{ parameters.packageVersion }}
BuildDropPath: ${{ parameters.buildDropPath }}
+ ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
publishArtifacts: false
# publish artifacts
@@ -30,6 +31,7 @@ jobs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
condition: always()
+ retryCountOnTaskFailure: 10 # for any logs being locked
continueOnError: true
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- output: pipelineArtifact
@@ -38,6 +40,7 @@ jobs:
displayName: 'Publish logs'
continueOnError: true
condition: always()
+ retryCountOnTaskFailure: 10 # for any logs being locked
sbomEnabled: false # we don't need SBOM for logs
- ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}:
@@ -45,7 +48,7 @@ jobs:
displayName: Publish Logs
PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
publishLocation: Container
- ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+ ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
continueOnError: true
condition: always()
sbomEnabled: false # we don't need SBOM for logs
diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml
index 100a3fc984..fcf6637b2e 100644
--- a/eng/common/templates-official/steps/publish-build-artifacts.yml
+++ b/eng/common/templates-official/steps/publish-build-artifacts.yml
@@ -24,6 +24,10 @@ parameters:
- name: is1ESPipeline
type: boolean
default: true
+
+- name: retryCountOnTaskFailure
+ type: string
+ default: 10
steps:
- ${{ if ne(parameters.is1ESPipeline, true) }}:
@@ -38,4 +42,5 @@ steps:
PathtoPublish: ${{ parameters.pathToPublish }}
${{ if parameters.artifactName }}:
ArtifactName: ${{ parameters.artifactName }}
-
+ ${{ if parameters.retryCountOnTaskFailure }}:
+ retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
diff --git a/eng/common/templates-official/steps/source-index-stage1-publish.yml b/eng/common/templates-official/steps/source-index-stage1-publish.yml
new file mode 100644
index 0000000000..9b8b80942b
--- /dev/null
+++ b/eng/common/templates-official/steps/source-index-stage1-publish.yml
@@ -0,0 +1,7 @@
+steps:
+- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
+ parameters:
+ is1ESPipeline: true
+
+ ${{ each parameter in parameters }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index d1aeb92fce..7cbf668c22 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -46,6 +46,7 @@ jobs:
artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
continueOnError: true
condition: always()
+ retryCountOnTaskFailure: 10 # for any logs being locked
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
parameters:
@@ -56,6 +57,7 @@ jobs:
displayName: 'Publish logs'
continueOnError: true
condition: always()
+ retryCountOnTaskFailure: 10 # for any logs being locked
sbomEnabled: false # we don't need SBOM for logs
- ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
@@ -66,7 +68,7 @@ jobs:
displayName: Publish Logs
pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)'
publishLocation: Container
- artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
+ artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)_Attempt$(System.JobAttempt)' ) }}
continueOnError: true
condition: always()
diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml
index 6428a98dfe..605e602e94 100644
--- a/eng/common/templates/steps/publish-build-artifacts.yml
+++ b/eng/common/templates/steps/publish-build-artifacts.yml
@@ -25,6 +25,10 @@ parameters:
type: string
default: 'Container'
+- name: retryCountOnTaskFailure
+ type: string
+ default: 10
+
steps:
- ${{ if eq(parameters.is1ESPipeline, true) }}:
- 'eng/common/templates cannot be referenced from a 1ES managed template': error
@@ -37,4 +41,6 @@ steps:
PublishLocation: ${{ parameters.publishLocation }}
PathtoPublish: ${{ parameters.pathToPublish }}
${{ if parameters.artifactName }}:
- ArtifactName: ${{ parameters.artifactName }}
\ No newline at end of file
+ ArtifactName: ${{ parameters.artifactName }}
+ ${{ if parameters.retryCountOnTaskFailure }}:
+ retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }}
diff --git a/eng/common/templates/steps/source-index-stage1-publish.yml b/eng/common/templates/steps/source-index-stage1-publish.yml
new file mode 100644
index 0000000000..182cec33a7
--- /dev/null
+++ b/eng/common/templates/steps/source-index-stage1-publish.yml
@@ -0,0 +1,7 @@
+steps:
+- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
+ parameters:
+ is1ESPipeline: false
+
+ ${{ each parameter in parameters }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index a46b6deb75..7373e53054 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -42,7 +42,7 @@
[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
# Enable repos to use a particular version of the on-line dotnet-install scripts.
-# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
+# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
# True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -262,7 +262,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
- $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
+ $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
Retry({
Write-Host "GET $uri"
@@ -383,8 +383,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here:
- # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.12.0
- $defaultXCopyMSBuildVersion = '17.12.0'
+ # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0
+ $defaultXCopyMSBuildVersion = '17.13.0'
if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@@ -604,14 +604,7 @@ function InitializeBuildTool() {
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
- # Use override if it exists - commonly set by source-build
- if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) {
- $initializeBuildToolFramework="net9.0"
- } else {
- $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
- }
-
- $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
+ $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net' }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@@ -620,7 +613,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
- $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
+ $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "netframework"; ExcludePrereleaseVS = $excludePrereleaseVS }
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1
@@ -779,8 +772,10 @@ function MSBuild() {
# new scripts need to work with old packages, so we need to look for the old names/versions
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
- (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
- (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
+
+ # This list doesn't need to be updated anymore and can eventually be removed.
+ (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.ArcadeLogging.dll')),
+ (Join-Path $basePath (Join-Path net9.0 'Microsoft.DotNet.Arcade.Sdk.dll')),
(Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path net8.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
)
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 1159726a10..d51f300c77 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true}
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
# Enable repos to use a particular version of the on-line dotnet-install scripts.
-# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
+# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh
dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
# True to use global NuGet cache instead of restoring packages to repository-local directory.
@@ -295,7 +295,7 @@ function with_retries {
function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
- local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
+ local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
if [[ ! -a "$install_script" ]]; then
mkdir -p "$root"
@@ -339,12 +339,6 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
- # use override if it exists - commonly set by source-build
- if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then
- _InitializeBuildToolFramework="net9.0"
- else
- _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}"
- fi
}
# Set RestoreNoHttpCache as a workaround for https://github.com/NuGet/Home/issues/3116
@@ -454,10 +448,12 @@ function MSBuild {
# new scripts need to work with old packages, so we need to look for the old names/versions
local selectedPath=
local possiblePaths=()
- possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
- possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.ArcadeLogging.dll" )
- possiblePaths+=( "$toolset_dir/net7.0/Microsoft.DotNet.Arcade.Sdk.dll" )
+ possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/net/Microsoft.DotNet.Arcade.Sdk.dll" )
+
+ # This list doesn't need to be updated anymore and can eventually be removed.
+ possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.ArcadeLogging.dll" )
+ possiblePaths+=( "$toolset_dir/net9.0/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/net8.0/Microsoft.DotNet.Arcade.Sdk.dll" )
for path in "${possiblePaths[@]}"; do
@@ -532,6 +528,12 @@ function GetDarc {
"$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version
}
+# Returns a full path to an Arcade SDK task project file.
+function GetSdkTaskProject {
+ taskName=$1
+ echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj"
+}
+
ResolvePath "${BASH_SOURCE[0]}"
_script_dir=`dirname "$_ResolvePath"`
diff --git a/global.json b/global.json
index 29c1f847dd..3916996dc9 100644
--- a/global.json
+++ b/global.json
@@ -1,9 +1,9 @@
{
"sdk": {
- "version": "9.0.102"
+ "version": "10.0.100-preview.3.25167.3"
},
"tools": {
- "dotnet": "9.0.102",
+ "dotnet": "10.0.100-preview.3.25167.3",
"runtimes": {
"dotnet": [
"6.0.4",
@@ -19,6 +19,6 @@
},
"msbuild-sdks": {
"MSBuild.SDK.SystemWeb": "4.0.88",
- "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25065.2"
+ "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25178.2"
}
}
diff --git a/test/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests.csproj b/test/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests.csproj
index 4d34da1e10..6747715aa1 100644
--- a/test/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests.csproj
+++ b/test/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests/Microsoft.AspNetCore.SystemWebAdapters.CoreServices.Tests.csproj
@@ -7,7 +7,6 @@
-