diff --git a/tools/devops/automation/scripts/bash/export-mono-filenames.sh b/tools/devops/automation/scripts/bash/export-mono-filenames.sh new file mode 100755 index 000000000000..7108e503a966 --- /dev/null +++ b/tools/devops/automation/scripts/bash/export-mono-filenames.sh @@ -0,0 +1,25 @@ +#!/bin/bash -eu + +env | sort + +set -o pipefail +IFS=$'\n\t ' + +FILE=$(pwd)/tmp.txt + +make -C "$BUILD_SOURCESDIRECTORY/xamarin-macios/tools/devops" print-variable-value-to-file FILE="$FILE" VARIABLE=MONO_IOS_FILENAME +MONO_IOS_FILENAME=$(cat "$FILE") +MONO_IOS_FILENAME=$(basename "$MONO_IOS_FILENAME" ".7z") + +make -C "$BUILD_SOURCESDIRECTORY/xamarin-macios/tools/devops" print-variable-value-to-file FILE="$FILE" VARIABLE=MONO_MAC_FILENAME +MONO_MAC_FILENAME=$(cat "$FILE") +MONO_MAC_FILENAME=$(basename "$MONO_MAC_FILENAME" ".7z") + +make -C "$BUILD_SOURCESDIRECTORY/xamarin-macios/tools/devops" print-variable-value-to-file FILE="$FILE" VARIABLE=MONO_MACCATALYST_FILENAME +MONO_MACCATALYST_FILENAME=$(cat "$FILE") +MONO_MACCATALYST_FILENAME=$(basename "$MONO_MACCATALYST_FILENAME" ".7z") + +# allow the rest of the build use the values +echo "##vso[task.setvariable variable=MONO_IOS_FILENAME;]$MONO_IOS_FILENAME" +echo "##vso[task.setvariable variable=MONO_MAC_FILENAME;]$MONO_MAC_FILENAME" +echo "##vso[task.setvariable variable=MONO_MACCATALYST_FILENAME;]$MONO_MACCATALYST_FILENAME" diff --git a/tools/devops/automation/templates/build/api-diff-process-results.yml b/tools/devops/automation/templates/build/api-diff-process-results.yml index babd857db696..4f65536d2b3a 100644 --- a/tools/devops/automation/templates/build/api-diff-process-results.yml +++ b/tools/devops/automation/templates/build/api-diff-process-results.yml @@ -1,97 +1,97 @@ # Job that downloads the change detection artifact and: # * Uploads the results to VSDrops # * Publishes results (as a comment) to GitHub - +# yamllint disable rule:line-length parameters: -- name: isPR - type: boolean + - name: isPR + type: boolean -- name: repositoryAlias - type: string - default: self + - name: repositoryAlias + type: string + default: self -- name: commit - type: string - default: HEAD + - name: commit + type: string + default: HEAD -- name: uploadPrefix - type: string - default: '$(MaciosUploadPrefix)' + - name: uploadPrefix + type: string + default: '$(MaciosUploadPrefix)' steps: -- template: ../common/checkout.yml - parameters: - isPR: ${{ parameters.isPR }} - repositoryAlias: ${{ parameters.repositoryAlias }} - commit: ${{ parameters.commit }} - -- pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 - displayName: 'Show Environment' - -- pwsh: | - if (Test-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -PathType Container) { - Remove-Item -Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -Force -Recurse - } - displayName: "Remove artifacts" - condition: always() - -# Download the change detection artifact -- task: DownloadPipelineArtifact@2 - displayName: 'Download change detection artifacts' - inputs: - patterns: '${{ parameters.uploadPrefix }}change-detection/change-detection.zip' - allowFailedBuilds: true - path: $(System.DefaultWorkingDirectory)/Artifacts - -# Unzip the change detection artifact -- task: ExtractFiles@1 - displayName: 'Decompress change detection artifacts' - inputs: - archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Artifacts/${{ parameters.uploadPrefix }}change-detection/change-detection.zip' - destinationFolder: '$(System.DefaultWorkingDirectory)/change-detection' - -# Upload the change detection results to vsdrops -- task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 - displayName: 'Publish change detection results to Artifact Services Drop' - continueOnError: true # don't let any failures here stop us - inputs: - dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' - dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-ChangeDetection-$(System.JobAttempt)' - buildNumber: 'xamarin-macios/detected-changes/$(Build.BuildNumber)/$(Build.BuildId)-$(System.JobAttempt)' - sourcePath: '$(System.DefaultWorkingDirectory)/change-detection/results/' - detailedLog: true - usePat: true - -# Process the github comment and publish it -- pwsh: | - Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 - $vsdropsChangeDetectionPrefix = "https://vsdrop.corp.microsoft.com/file/v1/xamarin-macios/detected-changes/$Env:BUILD_BUILDNUMBER/$Env:BUILD_BUILDID-$Env:SYSTEM_JOBATTEMPT/;/" - - $rootDirectory = Join-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY" "change-detection" "results" - - $inputContentsPath = Join-Path -Path $rootDirectory -ChildPath "gh-comment.md" - if (Test-Path $inputContentsPath -PathType leaf) { - $inputContents = Get-Content -Path $inputContentsPath -Raw - } else { - $inputContents = ":fire: Unable to find the contents for the comment: $inputContentsPath does not exist :fire" - } - - try { - $converted = Convert-Markdown -RootDirectory $rootDirectory -InputContents $inputContents -VSDropsPrefix $vsdropsChangeDetectionPrefix - } catch { - $converted = $inputContents + "`n`nUnable to convert markdown: $_`n`n" - } - $githubComments = New-GitHubCommentsObjectFromUrl -Url "$(Build.Repository.Uri)" -Token $Env:GITHUB_TOKEN -Hash $Env:COMMENT_HASH - $result = $githubComments.NewCommentFromMessage("", "", $converted) - displayName: 'Publish GitHub comment for change detection' - timeoutInMinutes: 10 - continueOnError: true # don't let any failures here stop us - condition: always() # We always want to post something on github - env: - GITHUB_TOKEN: $(GitHub.Token) - ${{ if eq(parameters.repositoryAlias, 'self') }}: - COMMENT_HASH: $(GIT_HASH) - ${{ else }}: - COMMENT_HASH: $(Build.SourceVersion) + - template: ../common/checkout.yml + parameters: + isPR: ${{ parameters.isPR }} + repositoryAlias: ${{ parameters.repositoryAlias }} + commit: ${{ parameters.commit }} + + - pwsh: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/show_env.ps1 + displayName: 'Show Environment' + + - pwsh: | + if (Test-Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -PathType Container) { + Remove-Item -Path "$Env:SYSTEM_DEFAULTWORKINGDIRECTORY/Artifacts" -Force -Recurse + } + displayName: "Remove artifacts" + condition: always() + + # Download the change detection artifact + - task: DownloadPipelineArtifact@2 + displayName: 'Download change detection artifacts' + inputs: + patterns: '${{ parameters.uploadPrefix }}change-detection/change-detection.zip' + allowFailedBuilds: true + path: $(System.DefaultWorkingDirectory)/Artifacts + + # Unzip the change detection artifact + - task: ExtractFiles@1 + displayName: 'Decompress change detection artifacts' + inputs: + archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Artifacts/${{ parameters.uploadPrefix }}change-detection/change-detection.zip' + destinationFolder: '$(System.DefaultWorkingDirectory)/change-detection' + + # Upload the change detection results to vsdrops + - task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 + displayName: 'Publish change detection results to Artifact Services Drop' + continueOnError: true # don't let any failures here stop us + inputs: + dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' + dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-ChangeDetection-$(System.JobAttempt)' + buildNumber: 'xamarin-macios/detected-changes/$(Build.BuildNumber)/$(Build.BuildId)-$(System.JobAttempt)' + sourcePath: '$(System.DefaultWorkingDirectory)/change-detection/results/' + detailedLog: true + usePat: true + + # Process the github comment and publish it + - pwsh: | + Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY\xamarin-macios\tools\devops\automation\scripts\MaciosCI.psd1 + $vsdropsChangeDetectionPrefix = "https://vsdrop.corp.microsoft.com/file/v1/xamarin-macios/detected-changes/$Env:BUILD_BUILDNUMBER/$Env:BUILD_BUILDID-$Env:SYSTEM_JOBATTEMPT/;/" + + $rootDirectory = Join-Path "$Env:BUILD_ARTIFACTSTAGINGDIRECTORY" "change-detection" "results" + + $inputContentsPath = Join-Path -Path $rootDirectory -ChildPath "gh-comment.md" + if (Test-Path $inputContentsPath -PathType leaf) { + $inputContents = Get-Content -Path $inputContentsPath -Raw + } else { + $inputContents = ":fire: Unable to find the contents for the comment: $inputContentsPath does not exist :fire" + } + + try { + $converted = Convert-Markdown -RootDirectory $rootDirectory -InputContents $inputContents -VSDropsPrefix $vsdropsChangeDetectionPrefix + } catch { + $converted = $inputContents + "`n`nUnable to convert markdown: $_`n`n" + } + $githubComments = New-GitHubCommentsObjectFromUrl -Url "$(Build.Repository.Uri)" -Token $Env:GITHUB_TOKEN -Hash $Env:COMMENT_HASH + $result = $githubComments.NewCommentFromMessage("", "", $converted) + displayName: 'Publish GitHub comment for change detection' + timeoutInMinutes: 10 + continueOnError: true # don't let any failures here stop us + condition: always() # We always want to post something on github + env: + GITHUB_TOKEN: $(GitHub.Token) + ${{ if eq(parameters.repositoryAlias, 'self') }}: + COMMENT_HASH: $(GIT_HASH) + ${{ else }}: + COMMENT_HASH: $(Build.SourceVersion) diff --git a/tools/devops/automation/templates/build/build.yml b/tools/devops/automation/templates/build/build.yml index 235a79ae8fca..beda9b0e7c8b 100644 --- a/tools/devops/automation/templates/build/build.yml +++ b/tools/devops/automation/templates/build/build.yml @@ -168,6 +168,32 @@ steps: xcodebuild -downloadPlatform iOS displayName: 'Provision simulators' +# downloding mono takes time and has been shown to be problematic when we have network issues. This is why we +# are using the cache. Mono does not get bump a lot of times if any (after dotnet) + +- bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/export-mono-filenames.sh + displayName: "Export mono download filenames" + workingDirectory: "$(Build.SourcesDirectory)/xamarin-macios" + timeoutInMinutes: 5 + +- task: Cache@2 + displayName: 'Cache iOS Mono download' + inputs: + key: "$(MONO_IOS_FILENAME)" + path: "$(Build.SourcesDirectory)/xamarin-macios/builds/downloads/$(MONO_IOS_FILENAME)" + +- task: Cache@2 + displayName: 'Cache macOS Mono download' + inputs: + key: "$(MONO_MAC_FILENAME)" + path: "$(Build.SourcesDirectory)/xamarin-macios/builds/downloads/$(MONO_MAC_FILENAME)" + +- task: Cache@2 + displayName: 'Cache MacCatalsyt Mono download' + inputs: + key: "$(MONO_MACCATALYST_FILENAME)" + path: "$(Build.SourcesDirectory)/xamarin-macios/builds/downloads/$(MONO_MACCATALYST_FILENAME)" + # Actual build of the project - bash: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/automation/scripts/bash/build-macios.sh name: build diff --git a/tools/devops/automation/templates/tests/build.yml b/tools/devops/automation/templates/tests/build.yml index c8f8ea80772a..87480545725f 100644 --- a/tools/devops/automation/templates/tests/build.yml +++ b/tools/devops/automation/templates/tests/build.yml @@ -145,45 +145,81 @@ steps: AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_P12: ${{ parameters.xqaCertPass }} AUTH_TOKEN_VSENG_XAMARIN_MAC_DEVICES_2_P12: ${{ parameters.xqaCertPass }} +# this could be a template, which would make everything a lot simpler BUT alas we found a bug in Azure Devops (bugs++). +# if we use a template here azure devops will throw an exception due to a foreign key constraint error in their PK_Task_tbl_PlanContext. +# The simplest way to workaround their bad code is to remove the template and that way there wont be a task issue. I suspect that +# the problem resides in the fact that this is a template within a template within a matrix. - bash: | sudo rm -Rf $(Build.SourcesDirectory)/artifacts displayName: "Remove artifacts" condition: always() -- task: DownloadPipelineArtifact@2 - displayName: Download PkgsVersions.json - inputs: - allowFailedBuilds: true - patterns: '**/PkgsVersions.json' - path: $(Build.SourcesDirectory)/artifacts - -- task: DownloadPipelineArtifact@2 - displayName: Download WorkloadRollback.json - inputs: - allowFailedBuilds: true - patterns: '**/WorkloadRollback.json' - path: $(Build.SourcesDirectory)/artifacts - -- task: DownloadPipelineArtifact@2 - displayName: Download packages - inputs: - allowFailedBuilds: true - artifactName: not-signed-package - path: $(Build.SourcesDirectory)/artifacts/not-signed-package - -- task: DownloadPipelineArtifact@2 - displayName: Download test libraries - inputs: - allowFailedBuilds: true - artifactName: package-test-libraries - path: $(Build.SourcesDirectory)/artifacts/package-test-libraries - +# use a diff step depending if we have been trigger by a pipeline of by a PR/Commit +- ${{ if or(contains(variables['Build.Reason'], 'ResourceTrigger'), contains(variables['Build.Reason'], 'BuildCompletion')) }}: + - download: macios + displayName: Download PkgsVersions.json + artifact: PkgsVersions + + - download: macios + displayName: Download WorkloadRollback.json + artifact: WorkloadRollback + + - download: macios + displayName: Download packages + artifact: not-signed-package + + - download: macios + displayName: Download test libraries + artifact: package-test-libraries + + - pwsh: | + Get-ChildItem -Path "$(Pipeline.Workspace)/macios" -Recurse -Force + displayName: 'Display downloads' + timeoutInMinutes: 5 + + # the default location when downloading is $(Pipeline.Workspace)// + - bash: | + source="$PIPELINE_WORKSPACE/macios" + destination="$BUILD_SOURCESDIRECTORY/artifacts" + # move all the files from the source to the destination + mv -fv "$source" "$destination" + displayName: Move artifacts to the expected location + +- ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: Download PkgsVersions.json + inputs: + allowFailedBuilds: true + patterns: '**/PkgsVersions.json' + path: $(Build.SourcesDirectory)/artifacts + + - task: DownloadPipelineArtifact@2 + displayName: Download WorkloadRollback.json + inputs: + allowFailedBuilds: true + patterns: '**/WorkloadRollback.json' + path: $(Build.SourcesDirectory)/artifacts + + - task: DownloadPipelineArtifact@2 + displayName: Download packages + inputs: + allowFailedBuilds: true + artifactName: not-signed-package + path: $(Build.SourcesDirectory)/artifacts/not-signed-package + + - task: DownloadPipelineArtifact@2 + displayName: Download test libraries + inputs: + allowFailedBuilds: true + artifactName: package-test-libraries + path: $(Build.SourcesDirectory)/artifacts/package-test-libraries + +# print the downloads to make our life easier on debug - pwsh: | Get-ChildItem -Path $(Build.SourcesDirectory)/artifacts -Recurse -Force displayName: 'Display downloads' timeoutInMinutes: 5 - - pwsh: | $dir = "$(Build.SourcesDirectory)/artifacts" $versionsPath = "$dir/${{ parameters.uploadPrefix }}PkgsVersions/PkgsVersions.json"