Apply expected file permissions to Linux files after Authenticode sig…

…ning (PowerShell#18643) * Upload linux artifacts as tar.gz to preserve linux file permissions * Fix conditions * Fix yaml * Fix paths for downloaded artifacts * Rename while downloading * Fix file names and paths * Fix tar compress * Add set-location before tar * Create dirs before expanding * Create dir before expanding * Fix conditions * Add logging * Export file permission to json * Fix for condition * Apply file permissions before packaging * Add logging * Fix issues * Add comments
CarloToso · Jan 15, 2023 · b5e40b4 · b5e40b4
1 parent 93d16be
commit b5e40b4
Show file tree

Hide file tree

Showing 5 changed files with 292 additions and 54 deletions.
diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1
@@ -669,7 +669,7 @@ function New-TarballPackage {
 
             try {
                 Push-Location -Path $Staging
-                tar $options $packagePath .
+                tar $options $packagePath *
             } finally {
                 Pop-Location
             }
@@ -4480,15 +4480,21 @@ function Invoke-AzDevOpsLinuxPackageCreation {
 
         switch ($BuildType) {
             'fxdependent' {
+                $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\linuxFilePermission.json"
+                Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}"
                 Start-PSPackage -Type 'fxdependent' @releaseTagParam -LTS:$LTS
             }
             'alpine' {
+                $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\linuxFilePermission.json"
+                Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}"
                 Start-PSPackage -Type 'tar-alpine' @releaseTagParam -LTS:$LTS
             }
             'rpm' {
                 Start-PSPackage -Type 'rpm' @releaseTagParam -LTS:$LTS
             }
             default {
+                $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\linuxFilePermission.json"
+                Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}"
                 Start-PSPackage @releaseTagParam -LTS:$LTS -Type 'deb', 'tar'
             }
         }
@@ -4498,6 +4504,9 @@ function Invoke-AzDevOpsLinuxPackageCreation {
 
             Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${minSizeLinuxBuildFolder}-meta\psoptions.json"
 
+            $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${minSizeLinuxBuildFolder}-meta\linuxFilePermission.json"
+            Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${minSizeLinuxBuildFolder}"
+
             Write-Verbose -Verbose "---- Min-Size ----"
             Write-Verbose -Verbose "options.Output: $($options.Output)"
             Write-Verbose -Verbose "options.Top $($options.Top)"
@@ -4507,14 +4516,20 @@ function Invoke-AzDevOpsLinuxPackageCreation {
             ## Create 'linux-arm' 'tar.gz' package.
             ## Note that 'linux-arm' can only be built on Ubuntu environment.
             Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm32LinuxBuildFolder}-meta\psoptions.json"
+            $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm32LinuxBuildFolder}-meta\linuxFilePermission.json"
+            Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm32LinuxBuildFolder}"
             Start-PSPackage -Type tar-arm @releaseTagParam -LTS:$LTS
 
             ## Create 'linux-arm64' 'tar.gz' package.
             ## Note that 'linux-arm64' can only be built on Ubuntu environment.
             Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm64LinuxBuildFolder}-meta\psoptions.json"
+            $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm64LinuxBuildFolder}-meta\linuxFilePermission.json"
+            Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm64LinuxBuildFolder}"
             Start-PSPackage -Type tar-arm64 @releaseTagParam -LTS:$LTS
         } elseif ($BuildType -eq 'rpm') {
             Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${amd64MarinerBuildFolder}-meta\psoptions.json"
+            $filePermissionFile = "${env:SYSTEM_ARTIFACTSDIRECTORY}\${amd64MarinerBuildFolder}-meta\linuxFilePermission.json"
+            Set-LinuxFilePermission -FilePath $filePermissionFile -RootPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${amd64MarinerBuildFolder}"
 
             Write-Verbose -Verbose "---- rpm-fxdependent ----"
             Write-Verbose -Verbose "options.Output: $($options.Output)"
@@ -4567,6 +4582,8 @@ function Invoke-AzDevOpsLinuxPackageBuild {
 
         $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${mainLinuxBuildFolder}"
         Start-PSBuild @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
+        Get-ChildItem -Path $buildFolder -Recurse -File | Export-LinuxFilePermission -FilePath "${buildFolder}-meta/linuxFilePermission.json" -RootPath ${buildFolder} -Force
+
         # Remove symbol files.
         Remove-Item "${buildFolder}\*.pdb" -Force
 
@@ -4587,18 +4604,21 @@ function Invoke-AzDevOpsLinuxPackageBuild {
             Start-PSBuild -Clean @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
             # Remove symbol files, xml document files.
             Remove-Item "${buildFolder}\*.pdb", "${buildFolder}\*.xml" -Force
+            Get-ChildItem -Path $buildFolder -Recurse -File | Export-LinuxFilePermission -FilePath "${buildFolder}-meta/linuxFilePermission.json" -RootPath ${buildFolder} -Force
 
             ## Build 'linux-arm' and create 'tar.gz' package for it.
             ## Note that 'linux-arm' can only be built on Ubuntu environment.
             $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm32LinuxBuildFolder}"
             Start-PSBuild -Configuration Release -Restore -Runtime linux-arm -PSModuleRestore @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
             # Remove symbol files.
             Remove-Item "${buildFolder}\*.pdb" -Force
+            Get-ChildItem -Path $buildFolder -Recurse -File | Export-LinuxFilePermission -FilePath "${buildFolder}-meta/linuxFilePermission.json" -RootPath ${buildFolder} -Force
 
             $buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm64LinuxBuildFolder}"
             Start-PSBuild -Configuration Release -Restore -Runtime linux-arm64 -PSModuleRestore @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
             # Remove symbol files.
             Remove-Item "${buildFolder}\*.pdb" -Force
+            Get-ChildItem -Path $buildFolder -Recurse -File | Export-LinuxFilePermission -FilePath "${buildFolder}-meta/linuxFilePermission.json" -RootPath ${buildFolder} -Force
         } elseif ($BuildType -eq 'rpm') {
             ## Build for Mariner
             $options = Get-PSOptions
@@ -4616,6 +4636,7 @@ function Invoke-AzDevOpsLinuxPackageBuild {
             Start-PSBuild -Clean @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
             # Remove symbol files, xml document files.
             Remove-Item "${buildFolder}\*.pdb", "${buildFolder}\*.xml" -Force
+            Get-ChildItem -Path $buildFolder -Recurse -File | Export-LinuxFilePermission -FilePath "${buildFolder}-meta/linuxFilePermission.json" -RootPath ${buildFolder} -Force
         }
     }
     catch {
@@ -4624,6 +4645,110 @@ function Invoke-AzDevOpsLinuxPackageBuild {
     }
 }
 
+<#
+    Apply the file permissions specified in the json file $FilePath to the files under $RootPath.
+    The format of the json file is like:
+
+    {
+        "System.Net.WebClient.dll": "744",
+        "Schemas/PSMaml/developer.xsd": "644",
+        "ref/System.Security.AccessControl.dll": "744",
+        "ref/System.IO.dll": "744",
+        "cs/Microsoft.CodeAnalysis.resources.dll": "744",
+        "Schemas/PSMaml/base.xsd": "644",
+        "Schemas/PSMaml/structureProcedure.xsd": "644",
+        "ref/System.Net.Security.dll": "744"
+    }
+#>
+function Set-LinuxFilePermission {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)] [string] $FilePath,
+        [Parameter(Mandatory)] [string] $RootPath
+    )
+
+    if (-not (Test-Path $FilePath)) {
+        throw "File does not exist: $FilePath"
+    }
+
+    if (-not (Test-Path $RootPath)) {
+        throw "File does not exist: $RootPath"
+    }
+
+    try {
+        Push-Location $RootPath
+        $filePermission = Get-Content $FilePath -Raw | ConvertFrom-Json -AsHashtable
+
+        Write-Verbose -Verbose -Message "Got file permission: $($filePermission.Count) for $FilePath"
+
+        $filePermission.GetEnumerator() | ForEach-Object {
+            $file = $_.Name
+            $permission = $_.Value
+            $fileFullName = Join-Path -Path $RootPath -ChildPath $file
+            Write-Verbose "Set permission $permission to $fileFullName" -Verbose
+            chmod $permission $fileFullName
+        }
+    }
+    finally {
+        Pop-Location
+    }
+}
+
+<#
+    Store the linux file permissions for all the files under root path $RootPath to the json file $FilePath.
+    The json file stores them as relative paths to the root.
+
+    The format of the json file is like:
+
+    {
+        "System.Net.WebClient.dll": "744",
+        "Schemas/PSMaml/developer.xsd": "644",
+        "ref/System.Security.AccessControl.dll": "744",
+        "ref/System.IO.dll": "744",
+        "cs/Microsoft.CodeAnalysis.resources.dll": "744",
+        "Schemas/PSMaml/base.xsd": "644",
+        "Schemas/PSMaml/structureProcedure.xsd": "644",
+        "ref/System.Net.Security.dll": "744"
+    }
+
+#>
+function Export-LinuxFilePermission {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)] [string] $FilePath,
+        [Parameter(Mandatory)] [string] $RootPath,
+        [Parameter(Mandatory, ValueFromPipeline = $true)] [System.IO.FileInfo[]] $InputObject,
+        [Parameter()] [switch] $Force
+    )
+
+    begin {
+        if (Test-Path $FilePath) {
+          if (-not $Force) {
+            throw "File '$FilePath' already exists."
+          }
+          else {
+            Remove-Item $FilePath -Force
+          }
+        }
+
+        $fileData = @{}
+    }
+
+    process {
+        foreach ($object in $InputObject) {
+            Write-Verbose "Processing $($object.FullName)"
+            # This gets the unix stat information for the file in the format that chmod expects, like '644'.
+            $filePerms = [convert]::ToString($object.unixstat.mode, 8).substring(3)
+            $relativePath = [System.IO.Path]::GetRelativePath($RootPath, $_.FullName)
+            $fileData.Add($relativePath, $filePerms)
+        }
+    }
+
+    end {
+        $fileData | ConvertTo-Json -Depth 10 | Out-File -FilePath $FilePath
+    }
+}
+
 enum PackageManifestResultStatus {
     Mismatch
     Match

diff --git a/tools/releaseBuild/azureDevOps/templates/linux-authenticode-sign.yml b/tools/releaseBuild/azureDevOps/templates/linux-authenticode-sign.yml
@@ -20,46 +20,82 @@ jobs:
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshLinuxBuild
-      path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuild
+      artifact: pwshLinuxBuild.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars
     displayName: Download deb build
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshLinuxBuildMinSize
-      path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize
+      artifact: pwshLinuxBuildMinSize.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars
     displayName: Download min-size build
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshLinuxBuildArm32
-      path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32
+      artifact: pwshLinuxBuildArm32.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars
     displayName: Download arm32 build
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshLinuxBuildArm64
-      path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64
+      artifact: pwshLinuxBuildArm64.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars
     displayName: Download arm64 build
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshMarinerBuildAmd64
-      path: $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64
+      artifact: pwshMarinerBuildAmd64.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars
     displayName: Download mariner build
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshLinuxBuildAlpine
-      path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine
+      artifact: pwshLinuxBuildAlpine.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildAlpine.tar.gz
     displayName: Download alpine build
 
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: pwshLinuxBuildFxdependent
-      path: $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent
+      artifact: pwshLinuxBuildFxdependent.tar.gz
+      path: $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildFxdependent.tar.gz
     displayName: Download fxdependent build
 
+  - pwsh: |
+      Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/linuxTars
+    displayName: Capture downloaded tars
+
+  - pwsh: |
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuild"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuild
+      Write-Verbose -Verbose "File permisions after expanding"
+      Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuild/pwsh | Select-Object -Property 'unixmode', 'size', 'name'
+
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildMinSize.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildMinSize.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildMinSize
+
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm32.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32 -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm32.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm32
+
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64 -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildArm64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildArm64
+
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshMarinerBuildAmd64.tar.gz to $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64 -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshMarinerBuildAmd64.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshMarinerBuildAmd64
+
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildAlpine.tar.gz/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildAlpine.tar.gz/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildAlpine
+
+      Write-Verbose -Verbose -Message "Expanding $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildFxdependent.tar.gz/pwshLinuxBuild.tar.gz to $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent"
+      New-Item -Path $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent -ItemType Directory
+      tar -xf $(Build.ArtifactStagingDirectory)/linuxTars/pwshLinuxBuildFxdependent.tar.gz/pwshLinuxBuild.tar.gz -C $(Build.ArtifactStagingDirectory)/pwshLinuxBuildFxdependent
+    displayName: Expand builds
+
   - template: SetVersionVariables.yml
     parameters:
       ReleaseTagVar: $(ReleaseTagVar)
@@ -116,12 +152,3 @@ jobs:
     parameters:
       binLocation: pwshLinuxBuildFxdependent
       buildPrefixName: 'PowerShell Linux Framework Dependent'
-
-  #- template: Sbom.yml@ComplianceRepo
-  #  parameters:
-  #    BuildDropPath: '$(System.ArtifactsDirectory)/$(BIN_LOCATION)'
-  #    Build_Repository_Uri: $(Github_Build_Repository_Uri)
-  #    displayName: ${{ parameters.buildName }} SBOM
-  #    PackageName: $(PACKAGE_NAME)
-  #    PackageVersion: $(Version)
-  #    sourceScanPath: '$(PowerShellRoot)/tools'