From 5ab4bd4f735c74b4846779a08467be962fe09461 Mon Sep 17 00:00:00 2001 From: OneBranch Resources Date: Sat, 20 Apr 2024 17:33:41 +0000 Subject: [PATCH 01/34] Adding OneBranch pipeline YAML config file for PowerShell-Snap-Official --- .pipelines/PowerShell-Snap-Official.yml | 87 +++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .pipelines/PowerShell-Snap-Official.yml diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml new file mode 100644 index 0000000..80a19ac --- /dev/null +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -0,0 +1,87 @@ +################################################################################# +# OneBranch Pipelines # +# This pipeline was created by EasyStart from a sample located at: # +# https://aka.ms/obpipelines/easystart/samples # +# Documentation: https://aka.ms/obpipelines # +# Yaml Schema: https://aka.ms/obpipelines/yaml/schema # +# Retail Tasks: https://aka.ms/obpipelines/tasks # +# Support: https://aka.ms/onebranchsup # +################################################################################# + +trigger: none + +variables: + CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning + LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0' # Docker image which is used to build the project https://aka.ms/obpipelines/containers + DEBIAN_FRONTEND: noninteractive + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.Official.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates + parameters: + cloudvault: # https://aka.ms/obpipelines/cloudvault + enabled: false # set to true to enable cloudvault + runmode: stage # linux can run CloudVault upload as a separate stage + dependsOn: linux_build + artifacts: + - drop_linux_stage_linux_job + + globalSdl: # https://aka.ms/obpipelines/sdl + # tsa: + # enabled: true # SDL results of non-official builds aren't uploaded to TSA by default. + # credscan: + # suppressionsFile: $(Build.SourcesDirectory)\.config\CredScanSuppressions.json + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + # suppression: + # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress + + stages: + - stage: linux_stage + jobs: + - job: linux_job + pool: + type: linux + + variables: # More settings at https://aka.ms/obpipelines/yaml/jobs + ob_outputDirectory: '$(Build.SourcesDirectory)/out' # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts + + steps: # These steps will be run in unrestricted container's network + - task: onebranch.pipeline.version@1 + displayName: 'Setup BuildNumber' + inputs: + system: 'RevisionCounter' + major: '1' + minor: '0' + exclude_commit: true + + - task: Bash@3 + displayName: 'Restore' + inputs: + filePath: '$(Build.SourcesDirectory)/restore.sh' + + - task: Bash@3 + displayName: 'Build' + inputs: + filePath: '$(Build.SourcesDirectory)/build.sh' + + - task: Bash@3 + displayName: 'Package' + inputs: + filePath: '$(Build.SourcesDirectory)/package.sh' + + - task: Bash@3 + displayName: 'Copy Extra Files' + inputs: + targetType: 'inline' + script: | + mkdir -p $(Build.SourcesDirectory)/out + cp -a $(Build.SourcesDirectory)/linux_deploy $(Build.SourcesDirectory)/out + cp $(Build.SourcesDirectory)/*.tgz $(Build.SourcesDirectory)/out + cp $(Build.SourcesDirectory)/*.tgz.sha256 $(Build.SourcesDirectory)/out \ No newline at end of file From f1eef2aa6ab05f1c739703d03ed1ed2eb8c88fea Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:05:53 -0700 Subject: [PATCH 02/34] Port build to OneBranch --- .pipelines/PowerShell-Snap-Official.yml | 74 ++++---- .pipelines/templates/pushPhase.yml | 179 ++++++++++++++++++ .../templates/releaseBuildAndPushStage.yml | 23 +++ .pipelines/templates/releaseBuildPhase.yml | 117 ++++++++++++ 4 files changed, 351 insertions(+), 42 deletions(-) create mode 100644 .pipelines/templates/pushPhase.yml create mode 100644 .pipelines/templates/releaseBuildAndPushStage.yml create mode 100644 .pipelines/templates/releaseBuildPhase.yml diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index 80a19ac..debfaa1 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -8,15 +8,33 @@ # Support: https://aka.ms/onebranchsup # ################################################################################# +parameters: +- name: release + type: string + displayName: | + Use `stable` to release by default. + + Use `private` if you want to create a branch on the store to test the package. + This will create a branch under the edge branch automatically that are difficult, but not impossible to find. + + `candidate`, `beta`, and `edge` are public but we don't have any existing meaning for these channels. + values: + - private + - stable + - candidate + - beta + - edge + default: stable + trigger: none variables: CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning - LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0' # Docker image which is used to build the project https://aka.ms/obpipelines/containers + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2004:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers DEBIAN_FRONTEND: noninteractive resources: - repositories: + repositories: - repository: templates type: git name: OneBranch.Pipelines/GovernedTemplates @@ -43,45 +61,17 @@ extends: # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress stages: - - stage: linux_stage - jobs: - - job: linux_job - pool: - type: linux - - variables: # More settings at https://aka.ms/obpipelines/yaml/jobs - ob_outputDirectory: '$(Build.SourcesDirectory)/out' # this directory is uploaded to pipeline artifacts, reddog and cloudvault. More info at https://aka.ms/obpipelines/artifacts - - steps: # These steps will be run in unrestricted container's network - - task: onebranch.pipeline.version@1 - displayName: 'Setup BuildNumber' - inputs: - system: 'RevisionCounter' - major: '1' - minor: '0' - exclude_commit: true - - - task: Bash@3 - displayName: 'Restore' - inputs: - filePath: '$(Build.SourcesDirectory)/restore.sh' - - - task: Bash@3 - displayName: 'Build' - inputs: - filePath: '$(Build.SourcesDirectory)/build.sh' + - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self + parameters: + channel: stable + release: ${{ parameters.release }} - - task: Bash@3 - displayName: 'Package' - inputs: - filePath: '$(Build.SourcesDirectory)/package.sh' + - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self + parameters: + channel: preview + release: ${{ parameters.release }} - - task: Bash@3 - displayName: 'Copy Extra Files' - inputs: - targetType: 'inline' - script: | - mkdir -p $(Build.SourcesDirectory)/out - cp -a $(Build.SourcesDirectory)/linux_deploy $(Build.SourcesDirectory)/out - cp $(Build.SourcesDirectory)/*.tgz $(Build.SourcesDirectory)/out - cp $(Build.SourcesDirectory)/*.tgz.sha256 $(Build.SourcesDirectory)/out \ No newline at end of file + - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self + parameters: + channel: lts + release: ${{ parameters.release }} \ No newline at end of file diff --git a/.pipelines/templates/pushPhase.yml b/.pipelines/templates/pushPhase.yml new file mode 100644 index 0000000..41d9ee5 --- /dev/null +++ b/.pipelines/templates/pushPhase.yml @@ -0,0 +1,179 @@ +parameters: + - name: channel + type: string + - name: release + default: 'private' + +jobs: +- template: ./Approval.yml + parameters: + displayName: 'Approve ${{ parameters.channel }}-${{ parameters.release }} upload' + instructions: | + Only approve one channel at a time or malware scanning will fail. + Approving this will trigger the upload of the snap as "${{ parameters.channel }}-${{ parameters.release }}" + jobName: 'push_approval' + +- job: scan + displayName: Scan ${{ parameters.channel }} + + pool: + type: linux + + variables: + - name: channel + value: ${{ parameters.channel }} + - name: SNAP_MOUNT + value: $(Agent.TempDirectory)/pwshSnap + - name: SNAP_COPY + value: $(Agent.TempDirectory)/pwshSnapCopy + - name: SNAP_TARGZ + value: $(Agent.TempDirectory)/pwshSnap.tar.gz + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + + + steps: + - checkout: self + path: $(repoFolder) + - checkout: ComplianceRepo + + - task: DownloadPipelineArtifact@2 + displayName: 'Download build files' + inputs: + targetPath: $(signOutPath) + artifact: drop_build_$(channel)_build_$(channel) + + - pwsh: | + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + if($snaps.Count -gt 1) + { + $snaps | out-string -width 100 | Write-Verbose -verbose + Write-Error "***More than one snap found***" -errorAction stop + } + $snap = $snaps[0].fullname + $mount = $env:SNAP_MOUNT + $null = new-item -Path $mount -ItemType Directory -Force + Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" + sudo mount -t squashfs -o ro $snap $mount + Write-Verbose -Verbose "get-childitem -recurse $mount" + get-childitem -recurse $mount + displayName: Unsquash Snap and capture contents + + # Copy to read write filesystem because scan tries to write back to this folder. + - pwsh: | + $mount = $env:SNAP_MOUNT + $readWriteCopy = $env:SNAP_COPY + Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" + $null = new-item -Path $readWriteCopy -ItemType Directory -Force + $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint + $exclude = @() + $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } + Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose + Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" + get-childitem -recurse $readWriteCopy + displayName: Copy to read/write fs + + - pwsh: | + Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose + tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} + copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose + displayName: tar the snap contents from the r/w copy + +- job: push + dependsOn: + - push_approval + - scan + + displayName: Push to ${{ parameters.release }} + + pool: + name: $(ubuntuPool) + demands: + - ImageOverride -equals ${{ parameters.vmImage }} + + variables: + - name: ReleaseTag + value: edge/default + - group: SnapLogin + - name: channel + value: ${{ parameters.channel }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + + steps: + - checkout: self + + - task: DownloadPipelineArtifact@2 + displayName: 'Download build files' + inputs: + targetPath: $(signOutPath) + artifact: drop_build_$(channel)_build_$(channel) + + - pwsh: | + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + if($snaps.Count -gt 1) + { + $snaps | out-string -width 100 | Write-Verbose -verbose + Write-Error "***More than one snap found***" -errorAction stop + } + displayName: Verify that only one snap was downloaded + + - pwsh: | + [string]$Branch=$env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + if('${{ parameters.release }}' -eq 'private') { + if($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + { + Write-verbose "release branch: $branchOnly" -verbose + $generatedBranch = ([guid]::NewGuid()).ToString().Replace('-','') + $releaseTag = "edge/$generatedBranch" + } + else + { + Write-verbose "non-release branch" -verbose + # Branch is named + $releaseTag = "edge/$branchOnly" + $releaseTag += ([guid]::NewGuid()).ToString().Replace('-','') + } + } else { + $releaseTag = "${{ parameters.release }}" + } + + $vstsCommandString = "vso[task.setvariable variable=ReleaseTag]$releaseTag" + Write-Verbose -Message "setting ReleaseTag to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set ReleaseTag Variable + + - pwsh: | + sudo chown root:root / + displayName: 'Make sure root owns root' + condition: succeeded() + + - pwsh: | + $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel + Write-Verbose -Verbose -Message "using Channel $channel" + sudo snap install snapcraft --classic "--channel=$channel" + condition: succeeded() + displayName: 'Install snapcraft' + retryCountOnTaskFailure: 2 + + - pwsh: | + $track = 'latest' + if('$(channel)' -eq 'lts') + { + $track = 'lts' + } + + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File | Select-Object -ExpandProperty FullName + + foreach($snap in $snaps) + { + Write-Verbose -Verbose -Message "Uploading $snap to $track/$(ReleaseTag)" + snapcraft upload --release "$track/$(ReleaseTag)" $snap + } + displayName: 'snapcraft upload' + retryCountOnTaskFailure: 1 + env: + SNAPCRAFT_STORE_CREDENTIALS: $(SNAPCRAFT_STORE_CREDENTIALS) diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml new file mode 100644 index 0000000..846af8b --- /dev/null +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -0,0 +1,23 @@ +parameters: + - name: channel + default: '' + - name: release + default: 'private' + +stages: +- stage: build_${{ parameters.channel }} + displayName: Build - ${{ parameters.channel }} + dependsOn: [] + jobs: + - template: ./releaseBuildPhase.yml + parameters: + channel: ${{ parameters.channel }} + +- stage: Push_${{ parameters.channel }} + displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} + dependsOn: build_${{ parameters.channel }} + jobs: + - template: ./pushPhase.yml + parameters: + channel: ${{ parameters.channel }} + release: ${{ parameters.release }} diff --git a/.pipelines/templates/releaseBuildPhase.yml b/.pipelines/templates/releaseBuildPhase.yml new file mode 100644 index 0000000..0a18dfd --- /dev/null +++ b/.pipelines/templates/releaseBuildPhase.yml @@ -0,0 +1,117 @@ +parameters: + - name: channel + default: 'stable' + - name: test + default: Yes + +jobs: + +- job: build_${{ parameters.channel }} + displayName: Build ${{ parameters.channel }} + variables: + - name: channel + value: ${{ parameters.channel }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + + pool: + type: linux + + steps: + + - powershell: | + sudo chown root:root / + displayName: 'Make sure root owns root' + condition: succeeded() + + - powershell: 'dir env:' + displayName: 'Capture Environment' + + - powershell: | + $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel + Write-Verbose -Verbose -Message "using Channel $channel" + sudo snap install snapcraft --classic "--channel=$channel" + displayName: Install snapcraft + condition: succeeded() + retryCountOnTaskFailure: 2 + + - bash: | + sudo apt update + displayName: Apt Update + condition: succeeded() + + - powershell: | + switch('$(channel)') { + 'stable' { + cd stable + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" + Write-Host $message + Write-Host "##$message" + } + 'lts' { + cd lts + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" + Write-Host $message + Write-Host "##$message" + } + 'preview' { + cd preview + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell-preview" + Write-Host $message + Write-Host "##$message" + } + default { + throw "Unknown channel: '$(channel)'" + } + } + + Write-Host "Building snap..." + # destructive-mode leaves files and packages on the machine + # This is okay because Azure DevOps Hosted machines are ephemeral. + Write-Verbose "snapcraft version: $(snapcraft --version)" -Verbose + snapcraft snap --debug --destructive-mode + Write-Host "finding snap..." + $snap = Get-ChildItem "*.snap" -Recurse -ErrorAction SilentlyContinue | Select-Object -ExpandProperty fullname -ErrorAction SilentlyContinue + if(!$snap) + { + throw "snap build failed" + } + else + { + $message = "vso[task.setvariable variable=PS_SNAP_PATH;]$snap" + Write-Host $message + Write-Host "##$message" + } + displayName: Build snap $(channel) + condition: and(succeeded(), ne(variables['$(channel)'],'no')) + + - powershell: | + Get-ChildItem env: + displayName: Capture env + condition: always() + + - powershell: | + Get-ChildItem /home/vsts/.local/state/snapcraft/log/*.log | foreach-object { + $name = $_.fullname + Write-Verbose -Verbose "uploading '$name'" + Write-Host "##vso[artifact.upload containerfolder=$(channel)-logs;artifactname=$(channel)-log]$name" + } + displayName: 'Upload $(channel) build log' + condition: always() + + - powershell: | + sudo snap install $(PS_SNAP_PATH) --classic --dangerous + displayName: 'Install $(channel) snap' + condition: succeeded() + + - powershell: | + &"$(PS_SNAP_EXEC)" -nologo -c '$psversiontable' + displayName: 'Test $(channel) snap' + condition: and(succeeded(), eq('${{ parameters.test }}','Yes')) + + # Uploads any packages as an artifact + - powershell: | + Copy-Item $(PS_SNAP_PATH) -Destination $(ob_outputDirectory)/ -verbose + displayName: Copy Artifact to output directory - $(channel) + condition: succeeded() + continueOnError: true From f5e4cc17e4ac5f46bde179838ada14d62b9590fb Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:07:09 -0700 Subject: [PATCH 03/34] port approval as is --- .pipelines/templates/Approval.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .pipelines/templates/Approval.yml diff --git a/.pipelines/templates/Approval.yml b/.pipelines/templates/Approval.yml new file mode 100644 index 0000000..27a5da2 --- /dev/null +++ b/.pipelines/templates/Approval.yml @@ -0,0 +1,30 @@ +parameters: + - name: displayName + type: string + - name: instructions + type: string + - name: jobName + type: string + - name: timeoutInMinutes + type: number + # 2 days + default: 2880 + - name: onTimeout + type: string + default: 'reject' + values: + - resume + - reject + +jobs: + - job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + pool: server + timeoutInMinutes: 4320 # job times out in 3 days + steps: + - task: ManualValidation@0 + displayName: ${{ parameters.displayName }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + inputs: + instructions: ${{ parameters.instructions }} + onTimeout: ${{ parameters.onTimeout }} From 2e51f93fdf26862d4a1a1b60b27623262126b58c Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:07:54 -0700 Subject: [PATCH 04/34] fix pool type --- .pipelines/templates/pushPhase.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pipelines/templates/pushPhase.yml b/.pipelines/templates/pushPhase.yml index 41d9ee5..9749aba 100644 --- a/.pipelines/templates/pushPhase.yml +++ b/.pipelines/templates/pushPhase.yml @@ -87,9 +87,7 @@ jobs: displayName: Push to ${{ parameters.release }} pool: - name: $(ubuntuPool) - demands: - - ImageOverride -equals ${{ parameters.vmImage }} + type: linux variables: - name: ReleaseTag From 5e741b84b6c494ab7cbbf619fc58bcac858e5993 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:09:00 -0700 Subject: [PATCH 05/34] fix approval pool --- .pipelines/templates/Approval.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/Approval.yml b/.pipelines/templates/Approval.yml index 27a5da2..4d9d476 100644 --- a/.pipelines/templates/Approval.yml +++ b/.pipelines/templates/Approval.yml @@ -19,7 +19,8 @@ parameters: jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} - pool: server + pool: + type: agentless timeoutInMinutes: 4320 # job times out in 3 days steps: - task: ManualValidation@0 From e0ff0c89b363085ea99fd02f87163e6047a0aef5 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:10:28 -0700 Subject: [PATCH 06/34] remove unused checkout --- .pipelines/templates/pushPhase.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/pushPhase.yml b/.pipelines/templates/pushPhase.yml index 9749aba..dab97f1 100644 --- a/.pipelines/templates/pushPhase.yml +++ b/.pipelines/templates/pushPhase.yml @@ -35,7 +35,6 @@ jobs: steps: - checkout: self path: $(repoFolder) - - checkout: ComplianceRepo - task: DownloadPipelineArtifact@2 displayName: 'Download build files' From 66734543aa972bbd174f2e03d43791cadd3756d9 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:27:42 -0700 Subject: [PATCH 07/34] setup SDL --- .config/tsaoptions.json | 11 +++++++ .pipelines/PowerShell-Snap-Official.yml | 38 ++++++++++++++++++------- 2 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 .config/tsaoptions.json diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json new file mode 100644 index 0000000..90562ac --- /dev/null +++ b/.config/tsaoptions.json @@ -0,0 +1,11 @@ +{ + "instanceUrl": "https://msazure.visualstudio.com", + "projectName": "One", + "areaPath": "One\\MGMT\\Compute\\Powershell\\Powershell\\PowerShell Core", + "notificationAliases": [ + "adityap@microsoft.com", + "dongbow@microsoft.com", + "pmeinecke@microsoft.com", + "tplunk@microsoft.com" + ] +} \ No newline at end of file diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index debfaa1..f6d46eb 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -49,16 +49,34 @@ extends: dependsOn: linux_build artifacts: - drop_linux_stage_linux_job - - globalSdl: # https://aka.ms/obpipelines/sdl - # tsa: - # enabled: true # SDL results of non-official builds aren't uploaded to TSA by default. - # credscan: - # suppressionsFile: $(Build.SourcesDirectory)\.config\CredScanSuppressions.json - policheck: - break: true # always break the build on policheck issues. You can disable it by setting to 'false' - # suppression: - # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress + customTags: 'ES365AIMigrationTooling' + globalSdl: + disableLegacyManifest: true + # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + compiled: + enabled: false + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + cg: + enabled: true + asyncSdl: # https://aka.ms/obpipelines/asyncsdl + enabled: true + forStages: [build_lts, build_stable, build_preview, Push_lts, Push_stable, Push_preview] + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + #suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + binskim: + enabled: false + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config\tsaoptions.json stages: - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self From 51519a60055b97362a6eceafde05f786bf29a806 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:27:55 -0700 Subject: [PATCH 08/34] setup snapd --- .pipelines/templates/InstallSnapd.yml | 11 +++++++++++ .pipelines/templates/pushPhase.yml | 2 ++ .pipelines/templates/releaseBuildPhase.yml | 7 +++---- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 .pipelines/templates/InstallSnapd.yml diff --git a/.pipelines/templates/InstallSnapd.yml b/.pipelines/templates/InstallSnapd.yml new file mode 100644 index 0000000..7470306 --- /dev/null +++ b/.pipelines/templates/InstallSnapd.yml @@ -0,0 +1,11 @@ +jobs: + + - bash: | + sudo apt update + displayName: Apt Update + condition: succeeded() + + - bash: | + sudo apt-get -y install snapd + displayName: Install snapd + condition: succeeded() diff --git a/.pipelines/templates/pushPhase.yml b/.pipelines/templates/pushPhase.yml index dab97f1..b527c18 100644 --- a/.pipelines/templates/pushPhase.yml +++ b/.pipelines/templates/pushPhase.yml @@ -148,6 +148,8 @@ jobs: displayName: 'Make sure root owns root' condition: succeeded() + - template: ./InstallSnapd.yml + - pwsh: | $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel Write-Verbose -Verbose -Message "using Channel $channel" diff --git a/.pipelines/templates/releaseBuildPhase.yml b/.pipelines/templates/releaseBuildPhase.yml index 0a18dfd..44f5ac9 100644 --- a/.pipelines/templates/releaseBuildPhase.yml +++ b/.pipelines/templates/releaseBuildPhase.yml @@ -24,6 +24,8 @@ jobs: displayName: 'Make sure root owns root' condition: succeeded() + - template: ./InstallSnapd.yml + - powershell: 'dir env:' displayName: 'Capture Environment' @@ -35,10 +37,7 @@ jobs: condition: succeeded() retryCountOnTaskFailure: 2 - - bash: | - sudo apt update - displayName: Apt Update - condition: succeeded() + - powershell: | switch('$(channel)') { From fc8c73beec40522e87b006cb4423c3ea0f55569e Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:29:01 -0700 Subject: [PATCH 09/34] fix steps template --- .pipelines/templates/InstallSnapd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/InstallSnapd.yml b/.pipelines/templates/InstallSnapd.yml index 7470306..a8ed4f5 100644 --- a/.pipelines/templates/InstallSnapd.yml +++ b/.pipelines/templates/InstallSnapd.yml @@ -1,4 +1,4 @@ -jobs: +steps: - bash: | sudo apt update From 2d8c46a9619d665040445efc6557cb197a769bda Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:53:26 -0700 Subject: [PATCH 10/34] use vm because snap won't run otherwise --- .pipelines/PowerShell-Snap-Official.yml | 10 +++++++--- .pipelines/templates/{pushPhase.yml => pushJob.yml} | 4 ++++ .pipelines/templates/releaseBuildAndPushStage.yml | 4 ++-- .../{releaseBuildPhase.yml => releaseBuildJob.yml} | 4 ++++ 4 files changed, 17 insertions(+), 5 deletions(-) rename .pipelines/templates/{pushPhase.yml => pushJob.yml} (98%) rename .pipelines/templates/{releaseBuildPhase.yml => releaseBuildJob.yml} (96%) diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index f6d46eb..148540b 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -29,9 +29,13 @@ parameters: trigger: none variables: - CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning - LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2004:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers - DEBIAN_FRONTEND: noninteractive + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2004:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers + - name: DEBIAN_FRONTEND + value: noninteractive + - group: poolNames resources: repositories: diff --git a/.pipelines/templates/pushPhase.yml b/.pipelines/templates/pushJob.yml similarity index 98% rename from .pipelines/templates/pushPhase.yml rename to .pipelines/templates/pushJob.yml index b527c18..9c4a075 100644 --- a/.pipelines/templates/pushPhase.yml +++ b/.pipelines/templates/pushJob.yml @@ -87,6 +87,10 @@ jobs: pool: type: linux + isCustom: true + name: $(ubuntuPool) + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure variables: - name: ReleaseTag diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml index 846af8b..bb8fe7c 100644 --- a/.pipelines/templates/releaseBuildAndPushStage.yml +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -9,7 +9,7 @@ stages: displayName: Build - ${{ parameters.channel }} dependsOn: [] jobs: - - template: ./releaseBuildPhase.yml + - template: ./releaseBuildJob.yml parameters: channel: ${{ parameters.channel }} @@ -17,7 +17,7 @@ stages: displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} dependsOn: build_${{ parameters.channel }} jobs: - - template: ./pushPhase.yml + - template: ./pushJob.yml parameters: channel: ${{ parameters.channel }} release: ${{ parameters.release }} diff --git a/.pipelines/templates/releaseBuildPhase.yml b/.pipelines/templates/releaseBuildJob.yml similarity index 96% rename from .pipelines/templates/releaseBuildPhase.yml rename to .pipelines/templates/releaseBuildJob.yml index 44f5ac9..16d7b42 100644 --- a/.pipelines/templates/releaseBuildPhase.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -16,6 +16,10 @@ jobs: pool: type: linux + isCustom: true + name: $(ubuntuPool) + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure steps: From e9255e58daaf3d4c0009ebc636355f0035711aee Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 11:56:26 -0700 Subject: [PATCH 11/34] use home env to get logs --- .pipelines/templates/releaseBuildJob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 16d7b42..7f98662 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -94,7 +94,7 @@ jobs: condition: always() - powershell: | - Get-ChildItem /home/vsts/.local/state/snapcraft/log/*.log | foreach-object { + Get-ChildItem $env:HOME/.local/state/snapcraft/log/*.log | foreach-object { $name = $_.fullname Write-Verbose -Verbose "uploading '$name'" Write-Host "##vso[artifact.upload containerfolder=$(channel)-logs;artifactname=$(channel)-log]$name" From 5e34d4d81ae83f012e8b1ad8a3ec8b3007022286 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 12:30:02 -0700 Subject: [PATCH 12/34] No lfs --- .pipelines/templates/pushJob.yml | 2 ++ .pipelines/templates/releaseBuildJob.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 9c4a075..dc3a642 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -35,6 +35,7 @@ jobs: steps: - checkout: self path: $(repoFolder) + lfs: false - task: DownloadPipelineArtifact@2 displayName: 'Download build files' @@ -103,6 +104,7 @@ jobs: steps: - checkout: self + lfs: false - task: DownloadPipelineArtifact@2 displayName: 'Download build files' diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 7f98662..0ad2efb 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -6,6 +6,9 @@ parameters: jobs: +- checkout: self + lfs: false + - job: build_${{ parameters.channel }} displayName: Build ${{ parameters.channel }} variables: From ded5aec8f512294abf9ada576a0e69149d6fd8fc Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 12:31:42 -0700 Subject: [PATCH 13/34] fix syntax --- .pipelines/templates/releaseBuildJob.yml | 192 +++++++++++------------ 1 file changed, 95 insertions(+), 97 deletions(-) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 0ad2efb..020452e 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -6,9 +6,6 @@ parameters: jobs: -- checkout: self - lfs: false - - job: build_${{ parameters.channel }} displayName: Build ${{ parameters.channel }} variables: @@ -25,99 +22,100 @@ jobs: - ImageOverride -equals PSMMSUbuntu20.04-Secure steps: - - - powershell: | - sudo chown root:root / - displayName: 'Make sure root owns root' - condition: succeeded() - - - template: ./InstallSnapd.yml - - - powershell: 'dir env:' - displayName: 'Capture Environment' - - - powershell: | - $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel - Write-Verbose -Verbose -Message "using Channel $channel" - sudo snap install snapcraft --classic "--channel=$channel" - displayName: Install snapcraft - condition: succeeded() - retryCountOnTaskFailure: 2 - - - - - powershell: | - switch('$(channel)') { - 'stable' { - cd stable - $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" - Write-Host $message - Write-Host "##$message" - } - 'lts' { - cd lts - $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" - Write-Host $message - Write-Host "##$message" + - checkout: self + lfs: false + + + - powershell: | + sudo chown root:root / + displayName: 'Make sure root owns root' + condition: succeeded() + + - template: ./InstallSnapd.yml + + - powershell: 'dir env:' + displayName: 'Capture Environment' + + - powershell: | + $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel + Write-Verbose -Verbose -Message "using Channel $channel" + sudo snap install snapcraft --classic "--channel=$channel" + displayName: Install snapcraft + condition: succeeded() + retryCountOnTaskFailure: 2 + + - powershell: | + switch('$(channel)') { + 'stable' { + cd stable + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" + Write-Host $message + Write-Host "##$message" + } + 'lts' { + cd lts + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell" + Write-Host $message + Write-Host "##$message" + } + 'preview' { + cd preview + $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell-preview" + Write-Host $message + Write-Host "##$message" + } + default { + throw "Unknown channel: '$(channel)'" } - 'preview' { - cd preview - $message = "vso[task.setvariable variable=PS_SNAP_EXEC;]powershell-preview" - Write-Host $message - Write-Host "##$message" - } - default { - throw "Unknown channel: '$(channel)'" } - } - - Write-Host "Building snap..." - # destructive-mode leaves files and packages on the machine - # This is okay because Azure DevOps Hosted machines are ephemeral. - Write-Verbose "snapcraft version: $(snapcraft --version)" -Verbose - snapcraft snap --debug --destructive-mode - Write-Host "finding snap..." - $snap = Get-ChildItem "*.snap" -Recurse -ErrorAction SilentlyContinue | Select-Object -ExpandProperty fullname -ErrorAction SilentlyContinue - if(!$snap) - { - throw "snap build failed" - } - else - { - $message = "vso[task.setvariable variable=PS_SNAP_PATH;]$snap" - Write-Host $message - Write-Host "##$message" - } - displayName: Build snap $(channel) - condition: and(succeeded(), ne(variables['$(channel)'],'no')) - - - powershell: | - Get-ChildItem env: - displayName: Capture env - condition: always() - - - powershell: | - Get-ChildItem $env:HOME/.local/state/snapcraft/log/*.log | foreach-object { - $name = $_.fullname - Write-Verbose -Verbose "uploading '$name'" - Write-Host "##vso[artifact.upload containerfolder=$(channel)-logs;artifactname=$(channel)-log]$name" - } - displayName: 'Upload $(channel) build log' - condition: always() - - - powershell: | - sudo snap install $(PS_SNAP_PATH) --classic --dangerous - displayName: 'Install $(channel) snap' - condition: succeeded() - - - powershell: | - &"$(PS_SNAP_EXEC)" -nologo -c '$psversiontable' - displayName: 'Test $(channel) snap' - condition: and(succeeded(), eq('${{ parameters.test }}','Yes')) - - # Uploads any packages as an artifact - - powershell: | - Copy-Item $(PS_SNAP_PATH) -Destination $(ob_outputDirectory)/ -verbose - displayName: Copy Artifact to output directory - $(channel) - condition: succeeded() - continueOnError: true + + Write-Host "Building snap..." + # destructive-mode leaves files and packages on the machine + # This is okay because Azure DevOps Hosted machines are ephemeral. + Write-Verbose "snapcraft version: $(snapcraft --version)" -Verbose + snapcraft snap --debug --destructive-mode + Write-Host "finding snap..." + $snap = Get-ChildItem "*.snap" -Recurse -ErrorAction SilentlyContinue | Select-Object -ExpandProperty fullname -ErrorAction SilentlyContinue + if(!$snap) + { + throw "snap build failed" + } + else + { + $message = "vso[task.setvariable variable=PS_SNAP_PATH;]$snap" + Write-Host $message + Write-Host "##$message" + } + displayName: Build snap $(channel) + condition: and(succeeded(), ne(variables['$(channel)'],'no')) + + - powershell: | + Get-ChildItem env: + displayName: Capture env + condition: always() + + - powershell: | + Get-ChildItem $env:HOME/.local/state/snapcraft/log/*.log | foreach-object { + $name = $_.fullname + Write-Verbose -Verbose "uploading '$name'" + Write-Host "##vso[artifact.upload containerfolder=$(channel)-logs;artifactname=$(channel)-log]$name" + } + displayName: 'Upload $(channel) build log' + condition: always() + + - powershell: | + sudo snap install $(PS_SNAP_PATH) --classic --dangerous + displayName: 'Install $(channel) snap' + condition: succeeded() + + - powershell: | + &"$(PS_SNAP_EXEC)" -nologo -c '$psversiontable' + displayName: 'Test $(channel) snap' + condition: and(succeeded(), eq('${{ parameters.test }}','Yes')) + + # Uploads any packages as an artifact + - powershell: | + Copy-Item $(PS_SNAP_PATH) -Destination $(ob_outputDirectory)/ -verbose + displayName: Copy Artifact to output directory - $(channel) + condition: succeeded() + continueOnError: true From 2daed7da3569e2f99568c2501bc1544aadd287fc Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 12:44:16 -0700 Subject: [PATCH 14/34] use OBP params to disable lfs --- .pipelines/PowerShell-Snap-Official.yml | 7 ++ .pipelines/templates/pushJob.yml | 147 +++++++++++------------ .pipelines/templates/releaseBuildJob.yml | 4 - 3 files changed, 79 insertions(+), 79 deletions(-) diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index 148540b..4c944c8 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -47,6 +47,13 @@ resources: extends: template: v2/OneBranch.Official.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates parameters: + git: + fetchDepth: 1 + # windows only feature + #longpaths: true + retryCount: 3 + # we don't use this and some of our agents doesn't have the feature installed + lfs: false cloudvault: # https://aka.ms/obpipelines/cloudvault enabled: false # set to true to enable cloudvault runmode: stage # linux can run CloudVault upload as a separate stage diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index dc3a642..130fa37 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -81,8 +81,8 @@ jobs: - job: push dependsOn: - - push_approval - - scan + - push_approval + - scan displayName: Push to ${{ parameters.release }} @@ -103,82 +103,79 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/out' steps: - - checkout: self - lfs: false - - - task: DownloadPipelineArtifact@2 - displayName: 'Download build files' - inputs: - targetPath: $(signOutPath) - artifact: drop_build_$(channel)_build_$(channel) - - - pwsh: | - $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File - if($snaps.Count -gt 1) - { - $snaps | out-string -width 100 | Write-Verbose -verbose - Write-Error "***More than one snap found***" -errorAction stop - } - displayName: Verify that only one snap was downloaded - - - pwsh: | - [string]$Branch=$env:BUILD_SOURCEBRANCH - $branchOnly = $Branch -replace '^refs/heads/'; - $branchOnly = $branchOnly -replace '[_\-]' - - if('${{ parameters.release }}' -eq 'private') { - if($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + - task: DownloadPipelineArtifact@2 + displayName: 'Download build files' + inputs: + targetPath: $(signOutPath) + artifact: drop_build_$(channel)_build_$(channel) + + - pwsh: | + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + if($snaps.Count -gt 1) { - Write-verbose "release branch: $branchOnly" -verbose - $generatedBranch = ([guid]::NewGuid()).ToString().Replace('-','') - $releaseTag = "edge/$generatedBranch" + $snaps | out-string -width 100 | Write-Verbose -verbose + Write-Error "***More than one snap found***" -errorAction stop } - else - { - Write-verbose "non-release branch" -verbose - # Branch is named - $releaseTag = "edge/$branchOnly" - $releaseTag += ([guid]::NewGuid()).ToString().Replace('-','') + displayName: Verify that only one snap was downloaded + + - pwsh: | + [string]$Branch=$env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + if('${{ parameters.release }}' -eq 'private') { + if($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + { + Write-verbose "release branch: $branchOnly" -verbose + $generatedBranch = ([guid]::NewGuid()).ToString().Replace('-','') + $releaseTag = "edge/$generatedBranch" + } + else + { + Write-verbose "non-release branch" -verbose + # Branch is named + $releaseTag = "edge/$branchOnly" + $releaseTag += ([guid]::NewGuid()).ToString().Replace('-','') + } + } else { + $releaseTag = "${{ parameters.release }}" } - } else { - $releaseTag = "${{ parameters.release }}" - } - - $vstsCommandString = "vso[task.setvariable variable=ReleaseTag]$releaseTag" - Write-Verbose -Message "setting ReleaseTag to $releaseTag" -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Set ReleaseTag Variable - - - pwsh: | - sudo chown root:root / - displayName: 'Make sure root owns root' - condition: succeeded() - - - template: ./InstallSnapd.yml - - - pwsh: | - $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel - Write-Verbose -Verbose -Message "using Channel $channel" - sudo snap install snapcraft --classic "--channel=$channel" - condition: succeeded() - displayName: 'Install snapcraft' - retryCountOnTaskFailure: 2 - - pwsh: | - $track = 'latest' - if('$(channel)' -eq 'lts') - { - $track = 'lts' - } + $vstsCommandString = "vso[task.setvariable variable=ReleaseTag]$releaseTag" + Write-Verbose -Message "setting ReleaseTag to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set ReleaseTag Variable + + - pwsh: | + sudo chown root:root / + displayName: 'Make sure root owns root' + condition: succeeded() + + - template: ./InstallSnapd.yml + + - pwsh: | + $channel = (Get-Content ./snapcraftConfig.json | ConvertFrom-Json).channel + Write-Verbose -Verbose -Message "using Channel $channel" + sudo snap install snapcraft --classic "--channel=$channel" + condition: succeeded() + displayName: 'Install snapcraft' + retryCountOnTaskFailure: 2 + + - pwsh: | + $track = 'latest' + if('$(channel)' -eq 'lts') + { + $track = 'lts' + } - $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File | Select-Object -ExpandProperty FullName + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File | Select-Object -ExpandProperty FullName - foreach($snap in $snaps) - { - Write-Verbose -Verbose -Message "Uploading $snap to $track/$(ReleaseTag)" - snapcraft upload --release "$track/$(ReleaseTag)" $snap - } - displayName: 'snapcraft upload' - retryCountOnTaskFailure: 1 - env: - SNAPCRAFT_STORE_CREDENTIALS: $(SNAPCRAFT_STORE_CREDENTIALS) + foreach($snap in $snaps) + { + Write-Verbose -Verbose -Message "Uploading $snap to $track/$(ReleaseTag)" + snapcraft upload --release "$track/$(ReleaseTag)" $snap + } + displayName: 'snapcraft upload' + retryCountOnTaskFailure: 1 + env: + SNAPCRAFT_STORE_CREDENTIALS: $(SNAPCRAFT_STORE_CREDENTIALS) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 020452e..20deb1d 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -22,10 +22,6 @@ jobs: - ImageOverride -equals PSMMSUbuntu20.04-Secure steps: - - checkout: self - lfs: false - - - powershell: | sudo chown root:root / displayName: 'Make sure root owns root' From 5ed1f65b1b67cdd24e853d6867f7f245ed17a5f6 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 12:44:33 -0700 Subject: [PATCH 15/34] create output dir --- .pipelines/templates/createOutputDirectory-linux.yml | 6 ++++++ .pipelines/templates/pushJob.yml | 4 ++++ .pipelines/templates/releaseBuildJob.yml | 2 ++ 3 files changed, 12 insertions(+) create mode 100644 .pipelines/templates/createOutputDirectory-linux.yml diff --git a/.pipelines/templates/createOutputDirectory-linux.yml b/.pipelines/templates/createOutputDirectory-linux.yml new file mode 100644 index 0000000..6f00b62 --- /dev/null +++ b/.pipelines/templates/createOutputDirectory-linux.yml @@ -0,0 +1,6 @@ +steps: + + - bash: | + mkdir -p -m a=rwx $(ob_outputDirectory) + displayName: Create $(ob_outputDirectory) + condition: succeeded() diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 130fa37..2435ae6 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -37,6 +37,8 @@ jobs: path: $(repoFolder) lfs: false + - template: ./createOutputDirectory-linux.yml + - task: DownloadPipelineArtifact@2 displayName: 'Download build files' inputs: @@ -103,6 +105,8 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/out' steps: + - template: ./createOutputDirectory-linux.yml + - task: DownloadPipelineArtifact@2 displayName: 'Download build files' inputs: diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 20deb1d..9af69a5 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -22,6 +22,8 @@ jobs: - ImageOverride -equals PSMMSUbuntu20.04-Secure steps: + - template: ./createOutputDirectory-linux.yml + - powershell: | sudo chown root:root / displayName: 'Make sure root owns root' From f0ed457aac73829b95a3374c73868c568c933817 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 12:55:55 -0700 Subject: [PATCH 16/34] Use lfs setting in checkout because OBP params doesn't work due to custom pool This partially reverts commit 2daed7da3569e2f99568c2501bc1544aadd287fc. --- .pipelines/templates/pushJob.yml | 97 ++++++++++++------------ .pipelines/templates/releaseBuildJob.yml | 3 + 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 2435ae6..449ec62 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -33,53 +33,53 @@ jobs: steps: - - checkout: self - path: $(repoFolder) - lfs: false - - - template: ./createOutputDirectory-linux.yml - - - task: DownloadPipelineArtifact@2 - displayName: 'Download build files' - inputs: - targetPath: $(signOutPath) - artifact: drop_build_$(channel)_build_$(channel) - - - pwsh: | - $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File - if($snaps.Count -gt 1) - { - $snaps | out-string -width 100 | Write-Verbose -verbose - Write-Error "***More than one snap found***" -errorAction stop - } - $snap = $snaps[0].fullname - $mount = $env:SNAP_MOUNT - $null = new-item -Path $mount -ItemType Directory -Force - Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" - sudo mount -t squashfs -o ro $snap $mount - Write-Verbose -Verbose "get-childitem -recurse $mount" - get-childitem -recurse $mount - displayName: Unsquash Snap and capture contents - - # Copy to read write filesystem because scan tries to write back to this folder. - - pwsh: | - $mount = $env:SNAP_MOUNT - $readWriteCopy = $env:SNAP_COPY - Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" - $null = new-item -Path $readWriteCopy -ItemType Directory -Force - $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint - $exclude = @() - $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } - Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose - Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" - get-childitem -recurse $readWriteCopy - displayName: Copy to read/write fs - - - pwsh: | - Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose - tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} - copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose - displayName: tar the snap contents from the r/w copy + - checkout: self + path: $(repoFolder) + lfs: false + + - template: ./createOutputDirectory-linux.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download build files' + inputs: + targetPath: $(signOutPath) + artifact: drop_build_$(channel)_build_$(channel) + + - pwsh: | + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + if($snaps.Count -gt 1) + { + $snaps | out-string -width 100 | Write-Verbose -verbose + Write-Error "***More than one snap found***" -errorAction stop + } + $snap = $snaps[0].fullname + $mount = $env:SNAP_MOUNT + $null = new-item -Path $mount -ItemType Directory -Force + Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" + sudo mount -t squashfs -o ro $snap $mount + Write-Verbose -Verbose "get-childitem -recurse $mount" + get-childitem -recurse $mount + displayName: Unsquash Snap and capture contents + + # Copy to read write filesystem because scan tries to write back to this folder. + - pwsh: | + $mount = $env:SNAP_MOUNT + $readWriteCopy = $env:SNAP_COPY + Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" + $null = new-item -Path $readWriteCopy -ItemType Directory -Force + $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint + $exclude = @() + $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } + Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose + Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" + get-childitem -recurse $readWriteCopy + displayName: Copy to read/write fs + + - pwsh: | + Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose + tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} + copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose + displayName: tar the snap contents from the r/w copy - job: push dependsOn: @@ -105,6 +105,9 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/out' steps: + - checkout: self + lfs: false + - template: ./createOutputDirectory-linux.yml - task: DownloadPipelineArtifact@2 diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 9af69a5..250fc39 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -22,6 +22,9 @@ jobs: - ImageOverride -equals PSMMSUbuntu20.04-Secure steps: + - checkout: self + lfs: false + - template: ./createOutputDirectory-linux.yml - powershell: | From c201c8cdf74a40ae342360e0a97e7c65c1aaa96a Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 13:04:40 -0700 Subject: [PATCH 17/34] Fix artifact upload for custom job --- .pipelines/templates/releaseBuildJob.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 250fc39..e270fcb 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -114,9 +114,17 @@ jobs: displayName: 'Test $(channel) snap' condition: and(succeeded(), eq('${{ parameters.test }}','Yes')) - # Uploads any packages as an artifact + # This is what we would need to do if this wasn't a custom job - powershell: | Copy-Item $(PS_SNAP_PATH) -Destination $(ob_outputDirectory)/ -verbose - displayName: Copy Artifact to output directory - $(channel) + displayName: Copy Artifact to OneBranch Job output directory - $(channel) + condition: succeeded() + continueOnError: true + + # Uploads any packages as an artifact + - powershell: | + $artifactName = "drop_build_$(channel)_build_$(channel)" + Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]$(PS_SNAP_PATH)" + displayName: Publish Artifact for custom Job - $(channel) condition: succeeded() continueOnError: true From e18ea237f8fe15af488da5c97565bcc73ed8ca2b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 13:18:51 -0700 Subject: [PATCH 18/34] move mount out of approval --- .pipelines/templates/pushJob.yml | 69 ----------------- .../templates/releaseBuildAndPushStage.yml | 11 ++- .pipelines/templates/scanJob.yml | 74 +++++++++++++++++++ 3 files changed, 84 insertions(+), 70 deletions(-) create mode 100644 .pipelines/templates/scanJob.yml diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 449ec62..6e66a80 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -13,78 +13,9 @@ jobs: Approving this will trigger the upload of the snap as "${{ parameters.channel }}-${{ parameters.release }}" jobName: 'push_approval' -- job: scan - displayName: Scan ${{ parameters.channel }} - - pool: - type: linux - - variables: - - name: channel - value: ${{ parameters.channel }} - - name: SNAP_MOUNT - value: $(Agent.TempDirectory)/pwshSnap - - name: SNAP_COPY - value: $(Agent.TempDirectory)/pwshSnapCopy - - name: SNAP_TARGZ - value: $(Agent.TempDirectory)/pwshSnap.tar.gz - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/out' - - - steps: - - checkout: self - path: $(repoFolder) - lfs: false - - - template: ./createOutputDirectory-linux.yml - - - task: DownloadPipelineArtifact@2 - displayName: 'Download build files' - inputs: - targetPath: $(signOutPath) - artifact: drop_build_$(channel)_build_$(channel) - - - pwsh: | - $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File - if($snaps.Count -gt 1) - { - $snaps | out-string -width 100 | Write-Verbose -verbose - Write-Error "***More than one snap found***" -errorAction stop - } - $snap = $snaps[0].fullname - $mount = $env:SNAP_MOUNT - $null = new-item -Path $mount -ItemType Directory -Force - Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" - sudo mount -t squashfs -o ro $snap $mount - Write-Verbose -Verbose "get-childitem -recurse $mount" - get-childitem -recurse $mount - displayName: Unsquash Snap and capture contents - - # Copy to read write filesystem because scan tries to write back to this folder. - - pwsh: | - $mount = $env:SNAP_MOUNT - $readWriteCopy = $env:SNAP_COPY - Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" - $null = new-item -Path $readWriteCopy -ItemType Directory -Force - $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint - $exclude = @() - $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } - Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose - Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" - get-childitem -recurse $readWriteCopy - displayName: Copy to read/write fs - - - pwsh: | - Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose - tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} - copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose - displayName: tar the snap contents from the r/w copy - - job: push dependsOn: - push_approval - - scan displayName: Push to ${{ parameters.release }} diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml index bb8fe7c..3141aef 100644 --- a/.pipelines/templates/releaseBuildAndPushStage.yml +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -13,10 +13,19 @@ stages: parameters: channel: ${{ parameters.channel }} -- stage: Push_${{ parameters.channel }} +- stage: scan_${{ parameters.channel }} displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} dependsOn: build_${{ parameters.channel }} jobs: + - template: ./scanJob.yml + parameters: + channel: ${{ parameters.channel }} + release: ${{ parameters.release }} + +- stage: Push_${{ parameters.channel }} + displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} + dependsOn: scan_${{ parameters.channel }} + jobs: - template: ./pushJob.yml parameters: channel: ${{ parameters.channel }} diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml new file mode 100644 index 0000000..fe91ffb --- /dev/null +++ b/.pipelines/templates/scanJob.yml @@ -0,0 +1,74 @@ +parameters: + - name: channel + type: string + +jobs: + +- job: scan + displayName: Scan ${{ parameters.channel }} + + pool: + type: linux + + variables: + - name: channel + value: ${{ parameters.channel }} + - name: SNAP_MOUNT + value: $(Agent.TempDirectory)/pwshSnap + - name: SNAP_COPY + value: $(Agent.TempDirectory)/pwshSnapCopy + - name: SNAP_TARGZ + value: $(Agent.TempDirectory)/pwshSnap.tar.gz + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/out' + + + steps: + - checkout: self + path: $(repoFolder) + lfs: false + + - template: ./createOutputDirectory-linux.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download build files' + inputs: + targetPath: $(signOutPath) + artifact: drop_build_$(channel)_build_$(channel) + + - pwsh: | + $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + if($snaps.Count -gt 1) + { + $snaps | out-string -width 100 | Write-Verbose -verbose + Write-Error "***More than one snap found***" -errorAction stop + } + $snap = $snaps[0].fullname + $mount = $env:SNAP_MOUNT + $null = new-item -Path $mount -ItemType Directory -Force + Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" + sudo mount -t squashfs -o ro $snap $mount + Write-Verbose -Verbose "get-childitem -recurse $mount" + get-childitem -recurse $mount + displayName: Unsquash Snap and capture contents + + # Copy to read write filesystem because scan tries to write back to this folder. + - pwsh: | + $mount = $env:SNAP_MOUNT + $readWriteCopy = $env:SNAP_COPY + Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" + $null = new-item -Path $readWriteCopy -ItemType Directory -Force + $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint + $exclude = @() + $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } + Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose + Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" + get-childitem -recurse $readWriteCopy + displayName: Copy to read/write fs + + - pwsh: | + Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose + tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} + copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose + displayName: tar the snap contents from the r/w copy + From 4724d77872220edb83450a2d28836976519964cb Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 13:19:57 -0700 Subject: [PATCH 19/34] oops --- .pipelines/templates/releaseBuildAndPushStage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml index 3141aef..fff7ce5 100644 --- a/.pipelines/templates/releaseBuildAndPushStage.yml +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -20,7 +20,6 @@ stages: - template: ./scanJob.yml parameters: channel: ${{ parameters.channel }} - release: ${{ parameters.release }} - stage: Push_${{ parameters.channel }} displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} From 3108a27eccef15d9d9711734db47b50c2f701717 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 13:20:37 -0700 Subject: [PATCH 20/34] fix scan stage display name --- .pipelines/templates/releaseBuildAndPushStage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml index fff7ce5..7b00fbc 100644 --- a/.pipelines/templates/releaseBuildAndPushStage.yml +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -14,7 +14,7 @@ stages: channel: ${{ parameters.channel }} - stage: scan_${{ parameters.channel }} - displayName: Push - ${{ parameters.channel }} to ${{ parameters.release }} + displayName: Scan - ${{ parameters.channel }} to ${{ parameters.release }} dependsOn: build_${{ parameters.channel }} jobs: - template: ./scanJob.yml From c567775f2744cf3965688abfdecb70d0d89fedaf Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 13:35:48 -0700 Subject: [PATCH 21/34] switch to mariner when using container --- .pipelines/PowerShell-Snap-Official.yml | 2 +- .pipelines/templates/scanJob.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index 4c944c8..c4869a5 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -32,7 +32,7 @@ variables: - name: CDP_DEFINITION_BUILD_COUNT value: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning - name: LinuxContainerImage - value: 'onebranch.azurecr.io/linux/ubuntu-2004:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 # Docker image which is used to build the project https://aka.ms/obpipelines/containers - name: DEBIAN_FRONTEND value: noninteractive - group: poolNames diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index fe91ffb..daa5b42 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -36,6 +36,10 @@ jobs: targetPath: $(signOutPath) artifact: drop_build_$(channel)_build_$(channel) + - pwsh: | + sudo tdnf -y install squashfs-tools + displayName: Install squashfs-tools + - pwsh: | $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File if($snaps.Count -gt 1) From 302b1dfded2e54181f3745e197bfd425701f6e10 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 13:58:11 -0700 Subject: [PATCH 22/34] misc fixes --- .pipelines/PowerShell-Snap-Official.yml | 2 +- .pipelines/templates/releaseBuildAndPushStage.yml | 2 +- .pipelines/templates/scanJob.yml | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index c4869a5..4422029 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -77,7 +77,7 @@ extends: enabled: true asyncSdl: # https://aka.ms/obpipelines/asyncsdl enabled: true - forStages: [build_lts, build_stable, build_preview, Push_lts, Push_stable, Push_preview] + forStages: [scan_lts, scan_stable, scan_preview] credscan: enabled: true scanFolder: $(Build.SourcesDirectory) diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml index 7b00fbc..b76b011 100644 --- a/.pipelines/templates/releaseBuildAndPushStage.yml +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -14,7 +14,7 @@ stages: channel: ${{ parameters.channel }} - stage: scan_${{ parameters.channel }} - displayName: Scan - ${{ parameters.channel }} to ${{ parameters.release }} + displayName: SDL - ${{ parameters.channel }} to ${{ parameters.release }} dependsOn: build_${{ parameters.channel }} jobs: - template: ./scanJob.yml diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index daa5b42..70fa7a8 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -24,10 +24,6 @@ jobs: steps: - - checkout: self - path: $(repoFolder) - lfs: false - - template: ./createOutputDirectory-linux.yml - task: DownloadPipelineArtifact@2 From 5af9736c8e2b3a8adf8c82dcfb9ce65c8e34c9c1 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 14:23:51 -0700 Subject: [PATCH 23/34] fix scan download dir --- .pipelines/templates/pushJob.yml | 6 ++++-- .pipelines/templates/scanJob.yml | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 6e66a80..8ea3ec6 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -34,6 +34,8 @@ jobs: value: ${{ parameters.channel }} - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/out' + - name: DOWNLOAD_DIRECTORY + value: '$(Build.ArtifactStagingDirectory)/down' steps: - checkout: self @@ -44,11 +46,11 @@ jobs: - task: DownloadPipelineArtifact@2 displayName: 'Download build files' inputs: - targetPath: $(signOutPath) + targetPath: $(DOWNLOAD_DIRECTORY) artifact: drop_build_$(channel)_build_$(channel) - pwsh: | - $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + $snaps = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.snap -recurse -File if($snaps.Count -gt 1) { $snaps | out-string -width 100 | Write-Verbose -verbose diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index 70fa7a8..a409769 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -21,7 +21,8 @@ jobs: value: $(Agent.TempDirectory)/pwshSnap.tar.gz - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/out' - + - name: DOWNLOAD_DIRECTORY + value: '$(Build.ArtifactStagingDirectory)/down' steps: - template: ./createOutputDirectory-linux.yml @@ -29,7 +30,7 @@ jobs: - task: DownloadPipelineArtifact@2 displayName: 'Download build files' inputs: - targetPath: $(signOutPath) + targetPath: $(DOWNLOAD_DIRECTORY) artifact: drop_build_$(channel)_build_$(channel) - pwsh: | @@ -37,7 +38,7 @@ jobs: displayName: Install squashfs-tools - pwsh: | - $snaps = Get-ChildItem $(System.ArtifactsDirectory)/*.snap -recurse -File + $snaps = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.snap -recurse -File if($snaps.Count -gt 1) { $snaps | out-string -width 100 | Write-Verbose -verbose From 1b63680c45c6cc8b2d3fcca48974ceea955ffe14 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 14:55:46 -0700 Subject: [PATCH 24/34] unsquash during build --- .../templates/releaseBuildAndPushStage.yml | 2 +- .pipelines/templates/releaseBuildJob.yml | 44 +++++++++++++++++++ .pipelines/templates/scanJob.yml | 43 +----------------- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/.pipelines/templates/releaseBuildAndPushStage.yml b/.pipelines/templates/releaseBuildAndPushStage.yml index b76b011..203bb3d 100644 --- a/.pipelines/templates/releaseBuildAndPushStage.yml +++ b/.pipelines/templates/releaseBuildAndPushStage.yml @@ -14,7 +14,7 @@ stages: channel: ${{ parameters.channel }} - stage: scan_${{ parameters.channel }} - displayName: SDL - ${{ parameters.channel }} to ${{ parameters.release }} + displayName: SDL - ${{ parameters.channel }} dependsOn: build_${{ parameters.channel }} jobs: - template: ./scanJob.yml diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index e270fcb..c3b8d96 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -13,6 +13,12 @@ jobs: value: ${{ parameters.channel }} - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/out' + - name: SNAP_MOUNT + value: $(Agent.TempDirectory)/pwshSnap + - name: SNAP_COPY + value: $(Agent.TempDirectory)/pwshSnapCopy + - name: SNAP_TARGZ + value: $(Agent.TempDirectory)/pwshSnap.tar.gz pool: type: linux @@ -128,3 +134,41 @@ jobs: displayName: Publish Artifact for custom Job - $(channel) condition: succeeded() continueOnError: true + + - pwsh: | + $snap = $(PS_SNAP_PATH) + $mount = $env:SNAP_MOUNT + $null = new-item -Path $mount -ItemType Directory -Force + Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" + sudo mount -t squashfs -o ro $snap $mount + Write-Verbose -Verbose "get-childitem -recurse $mount" + get-childitem -recurse $mount + displayName: Unsquash Snap and capture contents + + # Copy to read write filesystem because scan tries to write back to this folder. + - pwsh: | + $mount = $env:SNAP_MOUNT + $readWriteCopy = $env:SNAP_COPY + Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" + $null = new-item -Path $readWriteCopy -ItemType Directory -Force + $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint + $exclude = @() + $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } + Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose + Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" + get-childitem -recurse $readWriteCopy + displayName: Copy to read/write fs + + - pwsh: | + Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose + tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} + copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose + displayName: tar the snap contents from the r/w copy + + # Uploads any packages as an artifact + - powershell: | + $artifactName = "drop_build_$(channel)_build_$(channel)_targz" + Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]${env:SNAP_TARGZ}" + displayName: Publish Artifact for custom Job - $(channel) + condition: succeeded() + continueOnError: true \ No newline at end of file diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index a409769..06d432f 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -28,48 +28,9 @@ jobs: - template: ./createOutputDirectory-linux.yml - task: DownloadPipelineArtifact@2 - displayName: 'Download build files' + displayName: 'Download targz files' inputs: targetPath: $(DOWNLOAD_DIRECTORY) - artifact: drop_build_$(channel)_build_$(channel) + artifact: drop_build_$(channel)_build_$(channel)_targz - - pwsh: | - sudo tdnf -y install squashfs-tools - displayName: Install squashfs-tools - - - pwsh: | - $snaps = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.snap -recurse -File - if($snaps.Count -gt 1) - { - $snaps | out-string -width 100 | Write-Verbose -verbose - Write-Error "***More than one snap found***" -errorAction stop - } - $snap = $snaps[0].fullname - $mount = $env:SNAP_MOUNT - $null = new-item -Path $mount -ItemType Directory -Force - Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" - sudo mount -t squashfs -o ro $snap $mount - Write-Verbose -Verbose "get-childitem -recurse $mount" - get-childitem -recurse $mount - displayName: Unsquash Snap and capture contents - - # Copy to read write filesystem because scan tries to write back to this folder. - - pwsh: | - $mount = $env:SNAP_MOUNT - $readWriteCopy = $env:SNAP_COPY - Write-Verbose -Verbose "mount: $mount; copy: $readWriteCopy" - $null = new-item -Path $readWriteCopy -ItemType Directory -Force - $filesToExclude = Get-ChildItem -Path $mount -Recurse | Where-Object Attributes -match reparsepoint - $exclude = @() - $exclude += $filesToExclude | ForEach-Object { "$($_.Name)" + "*" } - Copy-Item -Path $mount -Destination $readWriteCopy\ -Exclude $exclude -Force -Recurse -Verbose - Write-Verbose -Verbose "get-childitem -recurse $readWriteCopy" - get-childitem -recurse $readWriteCopy - displayName: Copy to read/write fs - - - pwsh: | - Write-Verbose "tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY}" -Verbose - tar -czf ${env:SNAP_TARGZ} ${env:SNAP_COPY} - copy-item ${env:SNAP_TARGZ} -destination $(ob_outputDirectory)/ -verbose - displayName: tar the snap contents from the r/w copy From ada144ce9ea927e2656c0438791fbe9db75e6547 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 16:02:07 -0700 Subject: [PATCH 25/34] publish output dir --- .pipelines/templates/releaseBuildJob.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index c3b8d96..639f020 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -123,15 +123,9 @@ jobs: # This is what we would need to do if this wasn't a custom job - powershell: | Copy-Item $(PS_SNAP_PATH) -Destination $(ob_outputDirectory)/ -verbose - displayName: Copy Artifact to OneBranch Job output directory - $(channel) - condition: succeeded() - continueOnError: true - - # Uploads any packages as an artifact - - powershell: | $artifactName = "drop_build_$(channel)_build_$(channel)" - Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]$(PS_SNAP_PATH)" - displayName: Publish Artifact for custom Job - $(channel) + Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]$(ob_outputDirectory)" + displayName: Copy Artifact to OneBranch Job output directory and publish - $(channel) condition: succeeded() continueOnError: true From b965c99b61c80072d270391d007c5923d0fee920 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 16:41:50 -0700 Subject: [PATCH 26/34] make it a string --- .pipelines/templates/releaseBuildJob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 639f020..dbca3c6 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -130,7 +130,7 @@ jobs: continueOnError: true - pwsh: | - $snap = $(PS_SNAP_PATH) + $snap = '$(PS_SNAP_PATH)' $mount = $env:SNAP_MOUNT $null = new-item -Path $mount -ItemType Directory -Force Write-Verbose -Verbose "sudo mount -t squashfs -o ro $snap $mount" From 8fdd358c08bd4ba9e539e1f790046ae97c97096b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 16:56:52 -0700 Subject: [PATCH 27/34] complete the scan job --- .pipelines/templates/releaseBuildJob.yml | 2 +- .pipelines/templates/scanJob.yml | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index dbca3c6..140a1e2 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -163,6 +163,6 @@ jobs: - powershell: | $artifactName = "drop_build_$(channel)_build_$(channel)_targz" Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]${env:SNAP_TARGZ}" - displayName: Publish Artifact for custom Job - $(channel) + displayName: Publish tar.gz Artifact - $(channel) condition: succeeded() continueOnError: true \ No newline at end of file diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index 06d432f..c1fa5ba 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -13,12 +13,6 @@ jobs: variables: - name: channel value: ${{ parameters.channel }} - - name: SNAP_MOUNT - value: $(Agent.TempDirectory)/pwshSnap - - name: SNAP_COPY - value: $(Agent.TempDirectory)/pwshSnapCopy - - name: SNAP_TARGZ - value: $(Agent.TempDirectory)/pwshSnap.tar.gz - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/out' - name: DOWNLOAD_DIRECTORY @@ -33,4 +27,12 @@ jobs: targetPath: $(DOWNLOAD_DIRECTORY) artifact: drop_build_$(channel)_build_$(channel)_targz + - pwsh: | + $snaps = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.snap -recurse -File | Select-Object -ExpandProperty FullName + foreach($snap in $snaps) + { + $target = $(ob_outputDirectory) + tar -xvf $snap -C $target + } + displayName: 'Extract Tar.Gz' From 3aa52bec2a1438904a73d69deee74e90f77e3c21 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 17:18:15 -0700 Subject: [PATCH 28/34] fix tar extraction --- .pipelines/templates/scanJob.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index c1fa5ba..a8c3add 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -28,11 +28,12 @@ jobs: artifact: drop_build_$(channel)_build_$(channel)_targz - pwsh: | - $snaps = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.snap -recurse -File | Select-Object -ExpandProperty FullName + $targzs = Get-ChildItem $(DOWNLOAD_DIRECTORY)/*.tar.gz -recurse -File | Select-Object -ExpandProperty FullName - foreach($snap in $snaps) + foreach($targz in $targzs) { + Write-Verbose -Verbose "Extracting $targz" $target = $(ob_outputDirectory) - tar -xvf $snap -C $target + tar -xvf $targz -C $target } displayName: 'Extract Tar.Gz' From f910864b1018c062f919584aefc028c57917a536 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sat, 20 Apr 2024 18:08:28 -0700 Subject: [PATCH 29/34] fix variable set statement --- .pipelines/templates/scanJob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/scanJob.yml b/.pipelines/templates/scanJob.yml index a8c3add..d40e3d3 100644 --- a/.pipelines/templates/scanJob.yml +++ b/.pipelines/templates/scanJob.yml @@ -33,7 +33,7 @@ jobs: foreach($targz in $targzs) { Write-Verbose -Verbose "Extracting $targz" - $target = $(ob_outputDirectory) + $target = '$(ob_outputDirectory)' tar -xvf $targz -C $target } displayName: 'Extract Tar.Gz' From 8892355a448ed35d505defd8f717e3dd1d803b26 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 22 Apr 2024 14:37:13 -0700 Subject: [PATCH 30/34] remove manual approval --- .pipelines/templates/pushJob.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 8ea3ec6..2548567 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -5,18 +5,7 @@ parameters: default: 'private' jobs: -- template: ./Approval.yml - parameters: - displayName: 'Approve ${{ parameters.channel }}-${{ parameters.release }} upload' - instructions: | - Only approve one channel at a time or malware scanning will fail. - Approving this will trigger the upload of the snap as "${{ parameters.channel }}-${{ parameters.release }}" - jobName: 'push_approval' - - job: push - dependsOn: - - push_approval - displayName: Push to ${{ parameters.release }} pool: From b9fab1d148ce289f4500df00f39943142d9bd26b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 22 Apr 2024 14:49:11 -0700 Subject: [PATCH 31/34] Update tsaoptions.json --- .config/tsaoptions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json index 90562ac..bd2a6a0 100644 --- a/.config/tsaoptions.json +++ b/.config/tsaoptions.json @@ -8,4 +8,4 @@ "pmeinecke@microsoft.com", "tplunk@microsoft.com" ] -} \ No newline at end of file +} From 8c6b76d774130d3b9dfb61b4086c30a460e62a90 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 22 Apr 2024 14:49:54 -0700 Subject: [PATCH 32/34] Update .pipelines/PowerShell-Snap-Official.yml --- .pipelines/PowerShell-Snap-Official.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index 4422029..8f5c9b2 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -1,12 +1,3 @@ -################################################################################# -# OneBranch Pipelines # -# This pipeline was created by EasyStart from a sample located at: # -# https://aka.ms/obpipelines/easystart/samples # -# Documentation: https://aka.ms/obpipelines # -# Yaml Schema: https://aka.ms/obpipelines/yaml/schema # -# Retail Tasks: https://aka.ms/obpipelines/tasks # -# Support: https://aka.ms/onebranchsup # -################################################################################# parameters: - name: release From cbf70bbcc03df50b79aabb1791fb72aa97476d1c Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 22 Apr 2024 14:52:29 -0700 Subject: [PATCH 33/34] Apply suggestions from code review --- .pipelines/PowerShell-Snap-Official.yml | 10 +++++----- .pipelines/templates/releaseBuildJob.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pipelines/PowerShell-Snap-Official.yml b/.pipelines/PowerShell-Snap-Official.yml index 8f5c9b2..2d81bb6 100644 --- a/.pipelines/PowerShell-Snap-Official.yml +++ b/.pipelines/PowerShell-Snap-Official.yml @@ -21,9 +21,9 @@ trigger: none variables: - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning + value: $[counter('', 0)] # needed for onebranch.pipeline.version task - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 # Docker image which is used to build the project https://aka.ms/obpipelines/containers + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 # Docker image which is used to build the project - name: DEBIAN_FRONTEND value: noninteractive - group: poolNames @@ -36,7 +36,7 @@ resources: ref: refs/heads/main extends: - template: v2/OneBranch.Official.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates + template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: git: fetchDepth: 1 @@ -45,7 +45,7 @@ extends: retryCount: 3 # we don't use this and some of our agents doesn't have the feature installed lfs: false - cloudvault: # https://aka.ms/obpipelines/cloudvault + cloudvault: enabled: false # set to true to enable cloudvault runmode: stage # linux can run CloudVault upload as a separate stage dependsOn: linux_build @@ -94,4 +94,4 @@ extends: - template: /.pipelines/templates/releaseBuildAndPushStage.yml@self parameters: channel: lts - release: ${{ parameters.release }} \ No newline at end of file + release: ${{ parameters.release }} diff --git a/.pipelines/templates/releaseBuildJob.yml b/.pipelines/templates/releaseBuildJob.yml index 140a1e2..c3d81a4 100644 --- a/.pipelines/templates/releaseBuildJob.yml +++ b/.pipelines/templates/releaseBuildJob.yml @@ -165,4 +165,4 @@ jobs: Write-Host "##vso[artifact.upload containerfolder=${artifactName};artifactname=${artifactName}]${env:SNAP_TARGZ}" displayName: Publish tar.gz Artifact - $(channel) condition: succeeded() - continueOnError: true \ No newline at end of file + continueOnError: true From 3f1adceda96e7fa626cb4409667819cb5f210064 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 22 Apr 2024 15:13:38 -0700 Subject: [PATCH 34/34] Update .pipelines/templates/pushJob.yml Co-authored-by: Aditya Patwardhan --- .pipelines/templates/pushJob.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/pushJob.yml b/.pipelines/templates/pushJob.yml index 2548567..bcd3151 100644 --- a/.pipelines/templates/pushJob.yml +++ b/.pipelines/templates/pushJob.yml @@ -66,7 +66,8 @@ jobs: $releaseTag = "edge/$branchOnly" $releaseTag += ([guid]::NewGuid()).ToString().Replace('-','') } - } else { + } + else { $releaseTag = "${{ parameters.release }}" }