Skip to content

Commit

Permalink
Merge branch 'main' into xcode15.3-bump
Browse files Browse the repository at this point in the history
  • Loading branch information
dalexsoto committed May 16, 2024
2 parents 65bf265 + b477bac commit f2e5620
Show file tree
Hide file tree
Showing 4 changed files with 202 additions and 115 deletions.
25 changes: 25 additions & 0 deletions tools/devops/automation/scripts/bash/export-mono-filenames.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash -eu

env | sort

set -o pipefail
IFS=$'\n\t '

FILE=$(pwd)/tmp.txt

make -C "$BUILD_SOURCESDIRECTORY/xamarin-macios/tools/devops" print-variable-value-to-file FILE="$FILE" VARIABLE=MONO_IOS_FILENAME
MONO_IOS_FILENAME=$(cat "$FILE")
MONO_IOS_FILENAME=$(basename "$MONO_IOS_FILENAME" ".7z")

make -C "$BUILD_SOURCESDIRECTORY/xamarin-macios/tools/devops" print-variable-value-to-file FILE="$FILE" VARIABLE=MONO_MAC_FILENAME
MONO_MAC_FILENAME=$(cat "$FILE")
MONO_MAC_FILENAME=$(basename "$MONO_MAC_FILENAME" ".7z")

make -C "$BUILD_SOURCESDIRECTORY/xamarin-macios/tools/devops" print-variable-value-to-file FILE="$FILE" VARIABLE=MONO_MACCATALYST_FILENAME
MONO_MACCATALYST_FILENAME=$(cat "$FILE")
MONO_MACCATALYST_FILENAME=$(basename "$MONO_MACCATALYST_FILENAME" ".7z")

# allow the rest of the build use the values
echo "##vso[task.setvariable variable=MONO_IOS_FILENAME;]$MONO_IOS_FILENAME"
echo "##vso[task.setvariable variable=MONO_MAC_FILENAME;]$MONO_MAC_FILENAME"
echo "##vso[task.setvariable variable=MONO_MACCATALYST_FILENAME;]$MONO_MACCATALYST_FILENAME"
172 changes: 86 additions & 86 deletions tools/devops/automation/templates/build/api-diff-process-results.yml
Original file line number Diff line number Diff line change
@@ -1,97 +1,97 @@
# Job that downloads the change detection artifact and:
# * Uploads the results to VSDrops
# * Publishes results (as a comment) to GitHub

# yamllint disable rule:line-length
parameters:

- name: isPR
type: boolean
- name: isPR
type: boolean

- name: repositoryAlias
type: string
default: self
- name: repositoryAlias
type: string
default: self

- name: commit
type: string
default: HEAD
- name: commit
type: string
default: HEAD

- name: uploadPrefix
type: string
default: '$(MaciosUploadPrefix)'
- name: uploadPrefix
type: string
default: '$(MaciosUploadPrefix)'

steps:

- template: ../common/checkout.yml
parameters:
isPR: ${{ parameters.isPR }}
repositoryAlias: ${{ parameters.repositoryAlias }}
commit: ${{ parameters.commit }}

- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1
displayName: 'Show Environment'

- pwsh: |
if (Test-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -PathType Container) {
Remove-Item -Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -Force -Recurse
}
displayName: "Remove artifacts"
condition: always()

# Download the change detection artifact
- task: DownloadPipelineArtifact@2
displayName: 'Download change detection artifacts'
inputs:
patterns: '${{ parameters.uploadPrefix }}change-detection/change-detection.zip'
allowFailedBuilds: true
path: $(System.DefaultWorkingDirectory)/Artifacts

# Unzip the change detection artifact
- task: ExtractFiles@1
displayName: 'Decompress change detection artifacts'
inputs:
archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Artifacts/${{ parameters.uploadPrefix }}change-detection/change-detection.zip'
destinationFolder: '$(System.DefaultWorkingDirectory)/change-detection'

# Upload the change detection results to vsdrops
- task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0
displayName: 'Publish change detection results to Artifact Services Drop'
continueOnError: true # don't let any failures here stop us
inputs:
dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-ChangeDetection-$(System.JobAttempt)'
buildNumber: 'xamarin-macios/detected-changes/$(Build.BuildNumber)/$(Build.BuildId)-$(System.JobAttempt)'
sourcePath: '$(System.DefaultWorkingDirectory)/change-detection/results/'
detailedLog: true
usePat: true

# Process the github comment and publish it
- pwsh: |
Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1
$vsdropsChangeDetectionPrefix = "https://vsdrop.corp.microsoft.com/file/v1/xamarin-macios/detected-changes/$Env:BUILD_BUILDNUMBER/$Env:BUILD_BUILDID-$Env:SYSTEM_JOBATTEMPT/;/"
$rootDirectory = Join-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY" "change-detection" "results"
$inputContentsPath = Join-Path -Path $rootDirectory -ChildPath "gh-comment.md"
if (Test-Path $inputContentsPath -PathType leaf) {
$inputContents = Get-Content -Path $inputContentsPath -Raw
} else {
$inputContents = ":fire: Unable to find the contents for the comment: $inputContentsPath does not exist :fire"
}
try {
$converted = Convert-Markdown -RootDirectory $rootDirectory -InputContents $inputContents -VSDropsPrefix $vsdropsChangeDetectionPrefix
} catch {
$converted = $inputContents + "`n`nUnable to convert markdown: $_`n`n"
}
$githubComments = New-GitHubCommentsObjectFromUrl -Url "$(Build.Repository.Uri)" -Token $Env:GITHUB_TOKEN -Hash $Env:COMMENT_HASH
$result = $githubComments.NewCommentFromMessage("", "", $converted)
displayName: 'Publish GitHub comment for change detection'
timeoutInMinutes: 10
continueOnError: true # don't let any failures here stop us
condition: always() # We always want to post something on github
env:
GITHUB_TOKEN: $(GitHub.Token)
${{ if eq(parameters.repositoryAlias, 'self') }}:
COMMENT_HASH: $(GIT_HASH)
${{ else }}:
COMMENT_HASH: $(Build.SourceVersion)
- template: ../common/checkout.yml
parameters:
isPR: ${{ parameters.isPR }}
repositoryAlias: ${{ parameters.repositoryAlias }}
commit: ${{ parameters.commit }}

- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1
displayName: 'Show Environment'

- pwsh: |
if (Test-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -PathType Container) {
Remove-Item -Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -Force -Recurse
}
displayName: "Remove artifacts"
condition: always()
# Download the change detection artifact
- task: DownloadPipelineArtifact@2
displayName: 'Download change detection artifacts'
inputs:
patterns: '${{ parameters.uploadPrefix }}change-detection/change-detection.zip'
allowFailedBuilds: true
path: $(System.DefaultWorkingDirectory)/Artifacts

# Unzip the change detection artifact
- task: ExtractFiles@1
displayName: 'Decompress change detection artifacts'
inputs:
archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Artifacts/${{ parameters.uploadPrefix }}change-detection/change-detection.zip'
destinationFolder: '$(System.DefaultWorkingDirectory)/change-detection'

# Upload the change detection results to vsdrops
- task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0
displayName: 'Publish change detection results to Artifact Services Drop'
continueOnError: true # don't let any failures here stop us
inputs:
dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-ChangeDetection-$(System.JobAttempt)'
buildNumber: 'xamarin-macios/detected-changes/$(Build.BuildNumber)/$(Build.BuildId)-$(System.JobAttempt)'
sourcePath: '$(System.DefaultWorkingDirectory)/change-detection/results/'
detailedLog: true
usePat: true

# Process the github comment and publish it
- pwsh: |
Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1
$vsdropsChangeDetectionPrefix = "https://vsdrop.corp.microsoft.com/file/v1/xamarin-macios/detected-changes/$Env:BUILD_BUILDNUMBER/$Env:BUILD_BUILDID-$Env:SYSTEM_JOBATTEMPT/;/"
$rootDirectory = Join-Path "$Env:BUILD_ARTIFACTSTAGINGDIRECTORY" "change-detection" "results"
$inputContentsPath = Join-Path -Path $rootDirectory -ChildPath "gh-comment.md"
if (Test-Path $inputContentsPath -PathType leaf) {
$inputContents = Get-Content -Path $inputContentsPath -Raw
} else {
$inputContents = ":fire: Unable to find the contents for the comment: $inputContentsPath does not exist :fire"
}
try {
$converted = Convert-Markdown -RootDirectory $rootDirectory -InputContents $inputContents -VSDropsPrefix $vsdropsChangeDetectionPrefix
} catch {
$converted = $inputContents + "`n`nUnable to convert markdown: $_`n`n"
}
$githubComments = New-GitHubCommentsObjectFromUrl -Url "$(Build.Repository.Uri)" -Token $Env:GITHUB_TOKEN -Hash $Env:COMMENT_HASH
$result = $githubComments.NewCommentFromMessage("", "", $converted)
displayName: 'Publish GitHub comment for change detection'
timeoutInMinutes: 10
continueOnError: true # don't let any failures here stop us
condition: always() # We always want to post something on github
env:
GITHUB_TOKEN: $(GitHub.Token)
${{ if eq(parameters.repositoryAlias, 'self') }}:
COMMENT_HASH: $(GIT_HASH)
${{ else }}:
COMMENT_HASH: $(Build.SourceVersion)
26 changes: 26 additions & 0 deletions tools/devops/automation/templates/build/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,32 @@ steps:
xcodebuild -downloadPlatform iOS
displayName: 'Provision simulators'
# downloding mono takes time and has been shown to be problematic when we have network issues. This is why we
# are using the cache. Mono does not get bump a lot of times if any (after dotnet)

- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/export-mono-filenames.sh
displayName: "Export mono download filenames"
workingDirectory: "$(Build.SourcesDirectory)/xamarin-macios"
timeoutInMinutes: 5

- task: Cache@2
displayName: 'Cache iOS Mono download'
inputs:
key: "$(MONO_IOS_FILENAME)"
path: "$(Build.SourcesDirectory)/xamarin-macios/builds/downloads/$(MONO_IOS_FILENAME)"

- task: Cache@2
displayName: 'Cache macOS Mono download'
inputs:
key: "$(MONO_MAC_FILENAME)"
path: "$(Build.SourcesDirectory)/xamarin-macios/builds/downloads/$(MONO_MAC_FILENAME)"

- task: Cache@2
displayName: 'Cache MacCatalsyt Mono download'
inputs:
key: "$(MONO_MACCATALYST_FILENAME)"
path: "$(Build.SourcesDirectory)/xamarin-macios/builds/downloads/$(MONO_MACCATALYST_FILENAME)"

# Actual build of the project
- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/build-macios.sh
name: build
Expand Down
94 changes: 65 additions & 29 deletions tools/devops/automation/templates/tests/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,45 +145,81 @@ steps:
AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_P12: ${{ parameters.xqaCertPass }}
AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_2_P12: ${{ parameters.xqaCertPass }}

# this could be a template, which would make everything a lot simpler BUT alas we found a bug in Azure Devops (bugs++).
# if we use a template here azure devops will throw an exception due to a foreign key constraint error in their PK_Task_tbl_PlanContext.
# The simplest way to workaround their bad code is to remove the template and that way there wont be a task issue. I suspect that
# the problem resides in the fact that this is a template within a template within a matrix.
- bash: |
sudo rm -Rf $(Build.SourcesDirectory)/artifacts
displayName: "Remove artifacts"
condition: always()

- task: DownloadPipelineArtifact@2
displayName: Download PkgsVersions.json
inputs:
allowFailedBuilds: true
patterns: '**/PkgsVersions.json'
path: $(Build.SourcesDirectory)/artifacts

- task: DownloadPipelineArtifact@2
displayName: Download WorkloadRollback.json
inputs:
allowFailedBuilds: true
patterns: '**/WorkloadRollback.json'
path: $(Build.SourcesDirectory)/artifacts

- task: DownloadPipelineArtifact@2
displayName: Download packages
inputs:
allowFailedBuilds: true
artifactName: not-signed-package
path: $(Build.SourcesDirectory)/artifacts/not-signed-package

- task: DownloadPipelineArtifact@2
displayName: Download test libraries
inputs:
allowFailedBuilds: true
artifactName: package-test-libraries
path: $(Build.SourcesDirectory)/artifacts/package-test-libraries

# use a diff step depending if we have been trigger by a pipeline of by a PR/Commit
- ${{ if or(contains(variables['Build.Reason'], 'ResourceTrigger'), contains(variables['Build.Reason'], 'BuildCompletion')) }}:
- download: macios
displayName: Download PkgsVersions.json
artifact: PkgsVersions

- download: macios
displayName: Download WorkloadRollback.json
artifact: WorkloadRollback

- download: macios
displayName: Download packages
artifact: not-signed-package

- download: macios
displayName: Download test libraries
artifact: package-test-libraries

- pwsh: |
Get-ChildItem -Path "$(Pipeline.Workspace)/macios" -Recurse -Force
displayName: 'Display downloads'
timeoutInMinutes: 5
# the default location when downloading is $(Pipeline.Workspace)/<pipeline resource identifier>/<artifact name>
- bash: |
source="$PIPELINE_WORKSPACE/macios"
destination="$BUILD_SOURCESDIRECTORY/artifacts"
# move all the files from the source to the destination
mv -fv "$source" "$destination"
displayName: Move artifacts to the expected location
- ${{ else }}:
- task: DownloadPipelineArtifact@2
displayName: Download PkgsVersions.json
inputs:
allowFailedBuilds: true
patterns: '**/PkgsVersions.json'
path: $(Build.SourcesDirectory)/artifacts

- task: DownloadPipelineArtifact@2
displayName: Download WorkloadRollback.json
inputs:
allowFailedBuilds: true
patterns: '**/WorkloadRollback.json'
path: $(Build.SourcesDirectory)/artifacts

- task: DownloadPipelineArtifact@2
displayName: Download packages
inputs:
allowFailedBuilds: true
artifactName: not-signed-package
path: $(Build.SourcesDirectory)/artifacts/not-signed-package

- task: DownloadPipelineArtifact@2
displayName: Download test libraries
inputs:
allowFailedBuilds: true
artifactName: package-test-libraries
path: $(Build.SourcesDirectory)/artifacts/package-test-libraries

# print the downloads to make our life easier on debug
- pwsh: |
Get-ChildItem -Path $(Build.SourcesDirectory)/artifacts -Recurse -Force
displayName: 'Display downloads'
timeoutInMinutes: 5


- pwsh: |
$dir = "$(Build.SourcesDirectory)/artifacts"
$versionsPath = "$dir/${{ parameters.uploadPrefix }}PkgsVersions/PkgsVersions.json"
Expand Down

0 comments on commit f2e5620

Please sign in to comment.