Skip to content

Commit

Permalink
fixed extra data header, improved testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@saracen
saracen committed Nov 16, 2019
1 parent 4891785 commit 2af4ab5
Show file tree
Hide file tree
Showing 11 changed files with 134 additions and 59 deletions.
6 changes: 4 additions & 2 deletions .travis.yml
View file
Expand Up @@ -9,9 +9,11 @@ os:

go:
- 1.13.x
- master

go_import_path: github.com/saracen/fastzip

script:
- go test --cpu 1,4 -race ./...
- go test --cpu 1,4 -race -coverprofile=coverage.txt -covermode=atomic . ./internal/filepool/

after_success:
- bash <(curl -s https://codecov.io/bash)
53 changes: 27 additions & 26 deletions README.md
View file
Expand Up @@ -79,30 +79,31 @@ using the `WithArchiverConcurrency` and `WithExtractorConcurrency` options of 1,
goos: linux
goarch: amd64
pkg: github.com/saracen/fastzip
BenchmarkArchiveStore_1-24 2 767790415 ns/op 7546972 B/op 164417 allocs/op
BenchmarkArchiveStandardFlate_1-24 1 12701854697 ns/op 9031584 B/op 155352 allocs/op
BenchmarkArchiveStandardFlate_2-24 1 7231142160 ns/op 14654976 B/op 158651 allocs/op
BenchmarkArchiveStandardFlate_4-24 1 3529653713 ns/op 19643472 B/op 158806 allocs/op
BenchmarkArchiveStandardFlate_8-24 1 2065915120 ns/op 20600568 B/op 158941 allocs/op
BenchmarkArchiveStandardFlate_16-24 1 1585204206 ns/op 35697976 B/op 159616 allocs/op
BenchmarkArchiveNonStandardFlate_1-24 1 6639106839 ns/op 10377136 B/op 155331 allocs/op
BenchmarkArchiveNonStandardFlate_2-24 1 3882063711 ns/op 29038464 B/op 159414 allocs/op
BenchmarkArchiveNonStandardFlate_4-24 1 2064839564 ns/op 32078616 B/op 159452 allocs/op
BenchmarkArchiveNonStandardFlate_8-24 1 1077984082 ns/op 68054488 B/op 159777 allocs/op
BenchmarkArchiveNonStandardFlate_16-24 2 902048164 ns/op 69093092 B/op 159835 allocs/op
BenchmarkExtractStore_1-24 1 4937620997 ns/op 114333728 B/op 513607 allocs/op
BenchmarkExtractStore_2-24 1 2760969294 ns/op 114625376 B/op 513614 allocs/op
BenchmarkExtractStore_4-24 1 1370455758 ns/op 114699696 B/op 513628 allocs/op
BenchmarkExtractStore_8-24 2 931258382 ns/op 101096036 B/op 489593 allocs/op
BenchmarkExtractStore_16-24 2 769617330 ns/op 103773148 B/op 489937 allocs/op
BenchmarkExtractStandardFlate_1-24 1 5392231233 ns/op 114553664 B/op 513624 allocs/op
BenchmarkExtractStandardFlate_2-24 1 2880754249 ns/op 114626128 B/op 513622 allocs/op
BenchmarkExtractStandardFlate_4-24 1 1469073062 ns/op 115026176 B/op 513661 allocs/op
BenchmarkExtractStandardFlate_8-24 2 876169089 ns/op 102183440 B/op 489708 allocs/op
BenchmarkExtractStandardFlate_16-24 2 676284180 ns/op 104806528 B/op 490060 allocs/op
BenchmarkExtractNonStandardFlate_1-24 1 5260825440 ns/op 87035856 B/op 329816 allocs/op
BenchmarkExtractNonStandardFlate_2-24 1 2493055665 ns/op 87566864 B/op 332024 allocs/op
BenchmarkExtractNonStandardFlate_4-24 1 1608618424 ns/op 87261736 B/op 332567 allocs/op
BenchmarkExtractNonStandardFlate_8-24 2 923286287 ns/op 72071512 B/op 309891 allocs/op
BenchmarkExtractNonStandardFlate_16-24 2 642293998 ns/op 74520496 B/op 315397 allocs/op
BenchmarkArchiveStore_1-24 2 772482855 ns/op 8962844 B/op 257183 allocs/op
BenchmarkArchiveStandardFlate_1-24 1 13196699393 ns/op 9586992 B/op 248114 allocs/op
BenchmarkArchiveStandardFlate_2-24 1 7300203038 ns/op 15313704 B/op 251330 allocs/op
BenchmarkArchiveStandardFlate_4-24 1 3539821258 ns/op 24525648 B/op 251646 allocs/op
BenchmarkArchiveStandardFlate_8-24 1 1944286406 ns/op 19624032 B/op 251541 allocs/op
BenchmarkArchiveStandardFlate_16-24 1 1790596744 ns/op 29944856 B/op 252091 allocs/op
BenchmarkArchiveNonStandardFlate_1-24 1 6247745899 ns/op 14758776 B/op 248133 allocs/op
BenchmarkArchiveNonStandardFlate_2-24 1 3764029513 ns/op 33551632 B/op 252126 allocs/op
BenchmarkArchiveNonStandardFlate_4-24 1 1854558293 ns/op 27617888 B/op 252174 allocs/op
BenchmarkArchiveNonStandardFlate_8-24 1 1128922279 ns/op 75541672 B/op 252550 allocs/op
BenchmarkArchiveNonStandardFlate_16-24 2 945668228 ns/op 90173168 B/op 252723 allocs/op
BenchmarkExtractStore_1-24 1 5364033610 ns/op 116324288 B/op 565157 allocs/op
BenchmarkExtractStore_2-24 1 2639053970 ns/op 116255864 B/op 565142 allocs/op
BenchmarkExtractStore_4-24 1 1708197512 ns/op 116609776 B/op 565178 allocs/op
BenchmarkExtractStore_8-24 2 874622744 ns/op 103196552 B/op 541182 allocs/op
BenchmarkExtractStore_16-24 2 687568570 ns/op 106350048 B/op 541573 allocs/op
BenchmarkExtractStandardFlate_1-24 1 5142607436 ns/op 116689128 B/op 565209 allocs/op
BenchmarkExtractStandardFlate_2-24 1 2737468656 ns/op 116357072 B/op 565158 allocs/op
BenchmarkExtractStandardFlate_4-24 1 1351563778 ns/op 116649904 B/op 565178 allocs/op
BenchmarkExtractStandardFlate_8-24 2 996574326 ns/op 103007976 B/op 541164 allocs/op
BenchmarkExtractStandardFlate_16-24 2 658321262 ns/op 105358128 B/op 541463 allocs/op
BenchmarkExtractNonStandardFlate_1-24 1 5076043375 ns/op 88775520 B/op 381157 allocs/op
BenchmarkExtractNonStandardFlate_2-24 1 2714489120 ns/op 89336000 B/op 384490 allocs/op
BenchmarkExtractNonStandardFlate_4-24 1 1702744403 ns/op 89313920 B/op 384366 allocs/op
BenchmarkExtractNonStandardFlate_8-24 2 976734059 ns/op 74924428 B/op 362664 allocs/op
BenchmarkExtractNonStandardFlate_16-24 2 662083060 ns/op 75818612 B/op 365682 allocs/op
```
28 changes: 15 additions & 13 deletions archiver.go
View file
Expand Up @@ -149,22 +149,24 @@ func (a *Archiver) Archive(files map[string]os.FileInfo) (err error) {

switch hdr.Mode() & os.ModeType {
case os.ModeDir:
err = a.createDirectory(hdr)
err = a.createDirectory(fi, hdr)

case os.ModeSymlink:
err = a.createSymlink(path, hdr)
err = a.createSymlink(path, fi, hdr)

default:
hdr.Method = a.options.method
if hdr.UncompressedSize64 > 0 {
hdr.Method = a.options.method
}

if fp == nil {
err = a.createFile(path, hdr, nil)
err = a.createFile(path, fi, hdr, nil)
} else {
f := fp.Get()
wg.Go(func() error {
defer func() { fp.Put(f) }()

return a.createFile(path, hdr, f)
return a.createFile(path, fi, hdr, f)
})
}
}
Expand All @@ -177,19 +179,19 @@ func (a *Archiver) Archive(files map[string]os.FileInfo) (err error) {
return wg.Wait()
}

func (a *Archiver) createDirectory(hdr *zip.FileHeader) error {
func (a *Archiver) createDirectory(fi os.FileInfo, hdr *zip.FileHeader) error {
a.m.Lock()
defer a.m.Unlock()

_, err := a.createHeader(hdr)
_, err := a.createHeader(fi, hdr)
return err
}

func (a *Archiver) createSymlink(path string, hdr *zip.FileHeader) error {
func (a *Archiver) createSymlink(path string, fi os.FileInfo, hdr *zip.FileHeader) error {
a.m.Lock()
defer a.m.Unlock()

w, err := a.createHeader(hdr)
w, err := a.createHeader(fi, hdr)
if err != nil {
return err
}
Expand All @@ -203,7 +205,7 @@ func (a *Archiver) createSymlink(path string, hdr *zip.FileHeader) error {
return err
}

func (a *Archiver) createFile(path string, hdr *zip.FileHeader, tmp *filepool.File) (err error) {
func (a *Archiver) createFile(path string, fi os.FileInfo, hdr *zip.FileHeader, tmp *filepool.File) (err error) {
f, err := os.Open(path)
if err != nil {
return err
Expand All @@ -221,7 +223,7 @@ func (a *Archiver) createFile(path string, hdr *zip.FileHeader, tmp *filepool.Fi
a.m.Lock()
defer a.m.Unlock()

w, err := a.createHeader(hdr)
w, err := a.createHeader(fi, hdr)
if err != nil {
return err
}
Expand All @@ -247,14 +249,14 @@ func (a *Archiver) createFile(path string, hdr *zip.FileHeader, tmp *filepool.Fi
// if compressed file is larger, use the uncompressed version.
if hdr.CompressedSize64 > hdr.UncompressedSize64 {
hdr.Method = zip.Store
return a.createFile(path, hdr, nil)
return a.createFile(path, fi, hdr, nil)
}
hdr.CRC32 = tmp.Checksum()

a.m.Lock()
defer a.m.Unlock()

w, err := a.createHeaderRaw(hdr)
w, err := a.createHeaderRaw(fi, hdr)
if err != nil {
return err
}
Expand Down
8 changes: 7 additions & 1 deletion archiver_options.go
View file
Expand Up @@ -15,13 +15,16 @@ type archiverOptions struct {
stageDir string
}

// WithArchiverMethod sets the zip method to be used for compressible files.
func WithArchiverMethod(method uint16) ArchiverOption {
return func(o *archiverOptions) error {
o.method = method
return nil
}
}

// WithArchiverConcurrency will set the maximum number of files to be
// compressed concurrently. The default is GOMAXPROCS.
func WithArchiverConcurrency(n int) ArchiverOption {
return func(o *archiverOptions) error {
if n <= 0 {
Expand All @@ -32,7 +35,10 @@ func WithArchiverConcurrency(n int) ArchiverOption {
}
}

func WithStageDirectoryMethod(dir string) ArchiverOption {
// WithStageDirectory sets the directory to be used to stage compressed files
// before they're written to the archive. The default is the directory to be
// archived.
func WithStageDirectory(dir string) ArchiverOption {
return func(o *archiverOptions) error {
o.stageDir = dir
return nil
Expand Down
34 changes: 33 additions & 1 deletion archiver_test.go
View file
Expand Up @@ -91,6 +91,8 @@ func TestArchive(t *testing.T) {
"bar/foo/bar/foo/bar": testFile{mode: 0666},
"bar/symlink": testFile{mode: os.ModeDir | os.ModeSymlink | symMode, contents: "bar/foo/bar/foo"},
"bar/symlink.go": testFile{mode: os.ModeSymlink | symMode, contents: "foo/foo.go"},
"bar/compressible": testFile{mode: 0666, contents: "11111111111111111111111111111111111111111111111111"},
"bar/uncompressible": testFile{mode: 0666, contents: "A3#bez&OqCusPr)d&D]Vot9Eo0z^5O*VZm3:sO3HptL.H-4cOv"},
}

files, dir := testCreateFiles(t, testFiles)
Expand Down Expand Up @@ -146,6 +148,36 @@ func TestArchiveWithMethod(t *testing.T) {
testExtract(t, f.Name(), testFiles)
}

func TestArchiveWithStageDirectory(t *testing.T) {
testFiles := map[string]testFile{
"foo.go": testFile{mode: 0666},
"bar.go": testFile{mode: 0666},
}

files, chroot := testCreateFiles(t, testFiles)
defer os.RemoveAll(chroot)

dir, err := ioutil.TempDir("", "fastzip-benchmark-stage")
require.NoError(t, err)
defer os.RemoveAll(dir)

f, err := ioutil.TempFile("", "fastzip-test")
require.NoError(t, err)
defer os.Remove(f.Name())
defer f.Close()

a, err := NewArchiver(f, chroot, WithStageDirectory(dir))
require.NoError(t, err)
require.NoError(t, a.Archive(files))
require.NoError(t, a.Close())

stageFiles, err := ioutil.ReadDir(dir)
require.NoError(t, err)
require.Zero(t, len(stageFiles))

testExtract(t, f.Name(), testFiles)
}

func TestArchiveWithConcurrency(t *testing.T) {
testFiles := map[string]testFile{
"foo.go": testFile{mode: 0666},
Expand Down Expand Up @@ -247,7 +279,7 @@ func benchmarkArchiveOptions(b *testing.B, stdDeflate bool, options ...ArchiverO
require.NoError(b, err)
defer os.RemoveAll(dir)

options = append(options, WithStageDirectoryMethod(dir))
options = append(options, WithStageDirectory(dir))

b.ReportAllocs()
b.ResetTimer()
Expand Down
9 changes: 5 additions & 4 deletions archiver_unix.go
View file
Expand Up @@ -5,23 +5,24 @@ package fastzip
import (
"io"
"math/big"
"os"
"syscall"

"github.com/saracen/fastzip/internal/zip"
"github.com/saracen/zipextra"
)

func (a *Archiver) createHeader(hdr *zip.FileHeader) (io.Writer, error) {
stat, ok := hdr.FileInfo().Sys().(*syscall.Stat_t)
func (a *Archiver) createHeader(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) {
stat, ok := fi.Sys().(*syscall.Stat_t)
if ok {
hdr.Extra = append(hdr.Extra, zipextra.NewInfoZIPNewUnix(big.NewInt(int64(stat.Uid)), big.NewInt(int64(stat.Gid))).Encode()...)
}

return a.zw.CreateHeader(hdr)
}

func (a *Archiver) createHeaderRaw(hdr *zip.FileHeader) (io.Writer, error) {
stat, ok := hdr.FileInfo().Sys().(*syscall.Stat_t)
func (a *Archiver) createHeaderRaw(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) {
stat, ok := fi.Sys().(*syscall.Stat_t)
if ok {
hdr.Extra = append(hdr.Extra, zipextra.NewInfoZIPNewUnix(big.NewInt(int64(stat.Uid)), big.NewInt(int64(stat.Gid))).Encode()...)
}
Expand Down
5 changes: 3 additions & 2 deletions archiver_windows.go
View file
Expand Up @@ -4,14 +4,15 @@ package fastzip

import (
"io"
"os"

"github.com/saracen/fastzip/internal/zip"
)

func (a *Archiver) createHeader(hdr *zip.FileHeader) (io.Writer, error) {
func (a *Archiver) createHeader(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) {
return a.zw.CreateHeader(hdr)
}

func (a *Archiver) createHeaderRaw(hdr *zip.FileHeader) (io.Writer, error) {
func (a *Archiver) createHeaderRaw(fi os.FileInfo, hdr *zip.FileHeader) (io.Writer, error) {
return a.zw.CreateHeaderRaw(hdr)
}
9 changes: 2 additions & 7 deletions extractor.go
View file
Expand Up @@ -34,11 +34,6 @@ type Extractor struct {
m sync.Mutex
options extractorOptions
chroot string

// ChownErrorHandler handles errors that are encountered when trying to
// preserve ownership of extracted files. Returning nil will continue
// extraction, returning any error will cause Extract() to error.
ChownErrorHandler func(name string, err error) error
}

// NewExtractor returns a new extractor.
Expand Down Expand Up @@ -250,11 +245,11 @@ func (e *Extractor) updateFileMetadata(path string, file *zip.File) (err error)
}

if err := lchown(path, int(unix.Uid.Int64()), int(unix.Gid.Int64())); err != nil {
if e.ChownErrorHandler != nil {
if e.options.chownErrorHandler != nil {
e.m.Lock()
defer e.m.Unlock()

if err = e.ChownErrorHandler(file.Name, err); err != nil {
if err = e.options.chownErrorHandler(file.Name, err); err != nil {
return err
}
}
Expand Down
14 changes: 13 additions & 1 deletion extractor_options.go
View file
Expand Up @@ -4,7 +4,8 @@ package fastzip
type ExtractorOption func(*extractorOptions) error

type extractorOptions struct {
concurrency int
concurrency int
chownErrorHandler func(name string, err error) error
}

// WithExtractorConcurrency will set the maximum number of files being
Expand All @@ -18,3 +19,14 @@ func WithExtractorConcurrency(n int) ExtractorOption {
return nil
}
}

// WithExtractorChownErrorHandler sets an error handler to be called if errors are
// encountered when trying to preserve ownership of extracted files. Returning
// nil will continue extraction, returning any error will cause Extract() to
// error.
func WithExtractorChownErrorHandler(fn func(name string, err error) error) ExtractorOption {
return func(o *extractorOptions) error {
o.chownErrorHandler = fn
return nil
}
}
22 changes: 20 additions & 2 deletions extractor_test.go
View file
Expand Up @@ -64,7 +64,7 @@ func TestExtractorWithDecompressor(t *testing.T) {
})
}

func TestExtractorConcurrencyOption(t *testing.T) {
func TestExtractorWithConcurrency(t *testing.T) {
testFiles := map[string]testFile{
"foo.go": testFile{mode: 0666},
"bar.go": testFile{mode: 0666},
Expand Down Expand Up @@ -96,6 +96,24 @@ func TestExtractorConcurrencyOption(t *testing.T) {
})
}

func TestExtractorWithChownErrorHandler(t *testing.T) {
testFiles := map[string]testFile{
"foo.go": testFile{mode: 0666},
"bar.go": testFile{mode: 0666},
}

files, dir := testCreateFiles(t, testFiles)
defer os.RemoveAll(dir)

testCreateArchive(t, dir, files, func(filename, chroot string) {
e, err := NewExtractor(filename, dir, WithExtractorChownErrorHandler(func(name string, err error) error {
return nil
}))
assert.NoError(t, err)
require.NoError(t, e.Close())
})
}

func benchmarkExtractOptions(b *testing.B, stdDeflate bool, options ...ExtractorOption) {
files := make(map[string]os.FileInfo)
filepath.Walk(*archiveDir, func(filename string, fi os.FileInfo, err error) error {
Expand All @@ -112,7 +130,7 @@ func benchmarkExtractOptions(b *testing.B, stdDeflate bool, options ...Extractor
require.NoError(b, err)
defer os.Remove(f.Name())

a, err := NewArchiver(f, *archiveDir, WithStageDirectoryMethod(dir))
a, err := NewArchiver(f, *archiveDir, WithStageDirectory(dir))
a.RegisterCompressor(zip.Deflate, FlateCompressor(5))
require.NoError(b, err)

Expand Down

0 comments on commit 2af4ab5

Please sign in to comment.