Skip to content

Commit

Permalink
feat(decompress): Support Zstandard archive (#4372)
Browse files Browse the repository at this point in the history 
Co-authored-by: WorldYun <worldyun@qq.com>
Co-authored-by: Hsiao-nan Cheung <niheaven@gmail.com>
  • Loading branch information
@worldyun @niheaven
3 people committed Nov 10, 2021
1 parent 1490869 commit af7a6f6
Show file tree
Hide file tree
Showing 8 changed files with 136 additions and 19 deletions.
1 change: 1 addition & 0 deletions appveyor.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ environment:
scoop_helpers: C:\projects\helpers
lessmsi: '%scoop_helpers%\lessmsi\lessmsi.exe'
innounp: '%scoop_helpers%\innounp\innounp.exe'
zstd: '%scoop_helpers%\zstd\zstd.exe'
matrix:
- PowerShell: 5
- PowerShell: 6
Expand Down
5 changes: 3 additions & 2 deletions lib/core.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ function Get-HelperPath {
[CmdletBinding()]
param(
[Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true)]
[ValidateSet('7zip', 'Lessmsi', 'Innounp', 'Dark', 'Aria2')]
[ValidateSet('7zip', 'Lessmsi', 'Innounp', 'Dark', 'Aria2', 'Zstd')]
[String]
$Helper
)
Expand All @@ -265,6 +265,7 @@ function Get-HelperPath {
}
}
'Aria2' { $HelperPath = Get-AppFilePath 'aria2' 'aria2c.exe' }
'Zstd' { $HelperPath = Get-AppFilePath 'zstd' 'zstd.exe' }
}

return $HelperPath
Expand All @@ -274,7 +275,7 @@ function Test-HelperInstalled {
[CmdletBinding()]
param(
[Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true)]
[ValidateSet('7zip', 'Lessmsi', 'Innounp', 'Dark', 'Aria2')]
[ValidateSet('7zip', 'Lessmsi', 'Innounp', 'Dark', 'Aria2', 'Zstd')]
[String]
$Helper
)
Expand Down
71 changes: 70 additions & 1 deletion lib/decompress.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,25 @@ function Test-7zipRequirement {
return ($URL | Where-Object { Test-7zipRequirement -File $_ }).Count -gt 0
}
} else {
return $File -match '\.((gz)|(tar)|(tgz)|(lzma)|(bz)|(bz2)|(7z)|(rar)|(iso)|(xz)|(lzh)|(nupkg))$'
return $File -match '\.((gz)|(tar)|(t[abgpx]z2?)|(lzma)|(bz2?)|(7z)|(rar)|(iso)|(xz)|(lzh)|(nupkg))(\.[^.]+)?$'
}
}

function Test-ZstdRequirement {
[CmdletBinding(DefaultParameterSetName = "URL")]
[OutputType([Boolean])]
param (
[Parameter(Mandatory = $true, ParameterSetName = "URL")]
[String[]]
$URL,
[Parameter(Mandatory = $true, ParameterSetName = "File")]
[String]
$File
)
if ($URL) {
return ($URL | Where-Object { Test-ZstdRequirement -File $_ }).Count -gt 0
} else {
return $File -match '\.zst$'
}
}

Expand Down Expand Up @@ -104,6 +122,57 @@ function Expand-7zipArchive {
}
}

function Expand-ZstdArchive {
[CmdletBinding()]
param (
[Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true)]
[String]
$Path,
[Parameter(Position = 1)]
[String]
$DestinationPath = (Split-Path $Path),
[String]
$ExtractDir,
[Parameter(ValueFromRemainingArguments = $true)]
[String]
$Switches,
[Switch]
$Overwrite,
[Switch]
$Removal
)
$ZstdPath = Get-HelperPath -Helper Zstd
$LogPath = "$(Split-Path $Path)\zstd.log"
ensure $DestinationPath | Out-Null
$ArgList = @('-d', "`"$Path`"", '--output-dir-flat', "`"$DestinationPath`"")
if ($Switches) {
$ArgList += (-split $Switches)
}
if ($Overwrite) {
$ArgList += '-f'
}
$Status = Invoke-ExternalCommand $ZstdPath $ArgList -LogPath $LogPath
if (!$Status) {
abort "Failed to extract files from $Path.`nLog file:`n $(friendly_path $LogPath)`n$(new_issue_msg $app $bucket 'decompress error')"
}
$IsTar = (strip_ext $Path) -match '\.tar$'
if (!$IsTar -and $ExtractDir) {
movedir "$DestinationPath\$ExtractDir" $DestinationPath | Out-Null
}
if (Test-Path $LogPath) {
Remove-Item $LogPath -Force
}
if ($IsTar) {
# Check for tar
$TarFile = Get-ChildItem -Path $DestinationPath -Filter '*.tar'
Expand-7zipArchive -Path "$TarFile" -DestinationPath $DestinationPath -ExtractDir $ExtractDir -Removal
}
if ($Removal) {
# Remove original archive file
Remove-Item $Path -Force
}
}

function Expand-MsiArchive {
[CmdletBinding()]
param (
Expand Down
6 changes: 6 additions & 0 deletions lib/depends.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ function script_deps($script) {
if($script -like '*Expand-DarkArchive *') {
$deps += 'dark'
}
if ($script -like '*Expand-ZstdArchive *') {
$deps += 'zstd'
}

return $deps
}
Expand All @@ -86,6 +89,9 @@ function install_deps($manifest, $arch) {
if (!(Test-HelperInstalled -Helper Innounp) -and $manifest.innosetup) {
$deps += 'innounp'
}
if (!(Test-HelperInstalled -Helper Zstd) -and (Test-ZstdRequirement -URL (url $manifest $arch))) {
$deps += 'zstd'
}

$pre_install = arch_specific 'pre_install' $manifest $arch
$installer = arch_specific 'installer' $manifest $arch
Expand Down
2 changes: 2 additions & 0 deletions lib/install.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,8 @@ function dl_urls($app, $version, $manifest, $bucket, $architecture, $dir, $use_c
}
} elseif(Test-7zipRequirement -File $fname) { # 7zip
$extract_fn = 'Expand-7zipArchive'
} elseif(Test-ZstdRequirement -File $fname) { # Zstd
$extract_fn = 'Expand-ZstdArchive'
}

if($extract_fn) {
Expand Down
62 changes: 48 additions & 14 deletions test/Scoop-Decompress.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@ Describe 'Decompression function' -Tag 'Scoop', 'Decompress' {
It "Decompression test cases should exist" {
$testcases = "$working_dir\TestCases.zip"
$testcases | Should -Exist
compute_hash $testcases 'sha256' | Should -Be '695bb18cafda52644a19afd184b2545e9c48f1a191f7ff1efc26cb034587079c'
compute_hash $testcases 'sha256' | Should -Be '900b6e05275ad11a57dad85ffac6d9b4995657956a980bb1eae12c173f34a280'
if (!$isUnix) {
Microsoft.Powershell.Archive\Expand-Archive $testcases $working_dir
Microsoft.PowerShell.Archive\Expand-Archive $testcases $working_dir
}
}
}

Context "7zip extraction" {

BeforeAll {
if($env:CI) {
mock Get-AppFilePath { (Get-Command 7z.exe).Path }
} elseif(!(installed 7zip)) {
if ($env:CI) {
Mock Get-AppFilePath { (Get-Command 7z.exe).Path }
} elseif (!(installed 7zip)) {
scoop install 7zip
}
$test1 = "$working_dir\7ZipTest1.7z"
Expand Down Expand Up @@ -75,34 +75,68 @@ Describe 'Decompression function' -Tag 'Scoop', 'Decompress' {
}
}

Context "zstd extraction" {

BeforeAll {
if ($env:CI) {
Mock Get-AppFilePath { $env:zstd }
} elseif (!(installed zstd)) {
scoop install zstd
}

$test1 = "$working_dir\ZstdTest.zst"
$test2 = "$working_dir\ZstdTest.tar.zst"
}

It "extract normal compressed file" -Skip:$isUnix {
$to = test_extract "Expand-ZstdArchive" $test1
$to | Should -Exist
"$to\empty" | Should -Exist
(Get-ChildItem $to).Count | Should -Be 1
}

It "extract nested compressed file" -Skip:$isUnix {
$to = test_extract "Expand-7zipArchive" $test2
$to | Should -Exist
"$to\empty" | Should -Exist
(Get-ChildItem $to).Count | Should -Be 1
}

It "works with '-Removal' switch (`$removal param)" -Skip:$isUnix {
$test1 | Should -Exist
test_extract "Expand-ZstdArchive" $test1 $true
$test1 | Should -Not -Exist
}
}

Context "msi extraction" {

BeforeAll {
if($env:CI) {
mock Get-AppFilePath { $env:lessmsi }
} elseif(!(installed lessmsi)) {
if ($env:CI) {
Mock Get-AppFilePath { $env:lessmsi }
} elseif (!(installed lessmsi)) {
scoop install lessmsi
}
$test1 = "$working_dir\MSITest.msi"
$test2 = "$working_dir\MSITestNull.msi"
}

It "extract normal MSI file" -Skip:$isUnix {
mock get_config { $false }
Mock get_config { $false }
$to = test_extract "Expand-MsiArchive" $test1
$to | Should -Exist
"$to\MSITest\empty" | Should -Exist
(Get-ChildItem "$to\MSITest").Count | Should -Be 1
}

It "extract empty MSI file using lessmsi" -Skip:$isUnix {
mock get_config { $true }
Mock get_config { $true }
$to = test_extract "Expand-MsiArchive" $test2
$to | Should -Exist
}

It "works with '-Removal' switch (`$removal param)" -Skip:$isUnix {
mock get_config { $false }
Mock get_config { $false }
$test1 | Should -Exist
test_extract "Expand-MsiArchive" $test1 $true
$test1 | Should -Not -Exist
Expand All @@ -112,9 +146,9 @@ Describe 'Decompression function' -Tag 'Scoop', 'Decompress' {
Context "inno extraction" {

BeforeAll {
if($env:CI) {
mock Get-AppFilePath { $env:innounp }
} elseif(!(installed innounp)) {
if ($env:CI) {
Mock Get-AppFilePath { $env:innounp }
} elseif (!(installed innounp)) {
scoop install innounp
}
$test = "$working_dir\InnoTest.exe"
Expand Down
8 changes: 6 additions & 2 deletions test/bin/init.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,17 @@ if ($env:CI_WINDOWS -eq $true) {
New-Item -ItemType Directory -Path $env:SCOOP_HELPERS
}
if(!(Test-Path "$env:SCOOP_HELPERS\lessmsi\lessmsi.exe")) {
Start-FileDownload 'https://github.com/activescott/lessmsi/releases/download/v1.6.3/lessmsi-v1.6.3.zip' -FileName "$env:SCOOP_HELPERS\lessmsi.zip"
Start-FileDownload 'https://github.com/activescott/lessmsi/releases/download/v1.10.0/lessmsi-v1.10.0.zip' -FileName "$env:SCOOP_HELPERS\lessmsi.zip"
& 7z.exe x "$env:SCOOP_HELPERS\lessmsi.zip" -o"$env:SCOOP_HELPERS\lessmsi" -y
}
if(!(Test-Path "$env:SCOOP_HELPERS\innounp\innounp.exe")) {
Start-FileDownload 'https://raw.githubusercontent.com/ScoopInstaller/Binary/master/innounp/innounp048.rar' -FileName "$env:SCOOP_HELPERS\innounp.rar"
Start-FileDownload 'https://raw.githubusercontent.com/ScoopInstaller/Binary/master/innounp/innounp050.rar' -FileName "$env:SCOOP_HELPERS\innounp.rar"
& 7z.exe x "$env:SCOOP_HELPERS\innounp.rar" -o"$env:SCOOP_HELPERS\innounp" -y
}
if(!(Test-Path "$env:SCOOP_HELPERS\zstd\zstd.exe")) {
Start-FileDownload 'https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-v1.5.0-win64.zip' -FileName "$env:SCOOP_HELPERS\zstd.zip"
& 7z.exe x "$env:SCOOP_HELPERS\zstd.zip" "zstd-v1.5.0-win64" -o"$env:SCOOP_HELPERS\zstd" -y
}
}

if($env:CI -eq $true) {
Expand Down
Binary file modified test/fixtures/decompress/TestCases.zip
View file
Binary file not shown.

0 comments on commit af7a6f6

Please sign in to comment.