From c4c42105bb552375510d62762995f27acd1de9ec Mon Sep 17 00:00:00 2001 From: Songmu Date: Tue, 5 Mar 2019 02:13:45 +0900 Subject: [PATCH 1/4] add ziptree package for sharing code around compressing --- fs/fs_test.go | 46 +++--------------------- statik.go | 77 +++++++++------------------------------- ziptree/zipper.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 102 deletions(-) create mode 100644 ziptree/zipper.go diff --git a/fs/fs_test.go b/fs/fs_test.go index 36148db..789ebb3 100644 --- a/fs/fs_test.go +++ b/fs/fs_test.go @@ -15,18 +15,17 @@ package fs import ( "archive/zip" - "bytes" "io" "io/ioutil" "os" "path" - "path/filepath" "reflect" "sort" - "strings" "sync" "testing" "time" + + "github.com/rakyll/statik/ziptree" ) type wantFile struct { @@ -373,46 +372,11 @@ func BenchmarkOpen(b *testing.B) { // mustZipTree walks on the source path and returns the zipped file contents // as a string. Panics on any errors. func mustZipTree(srcPath string) string { - var out bytes.Buffer - w := zip.NewWriter(&out) - if err := filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - // Ignore directories and hidden files. - // No entry is needed for directories in a zip file. - // Each file is represented with a path, no directory - // entities are required to build the hierarchy. - if fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { - return nil - } - relPath, err := filepath.Rel(srcPath, path) - if err != nil { - return err - } - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - fHeader, err := zip.FileInfoHeader(fi) - if err != nil { - return err - } - fHeader.Name = filepath.ToSlash(relPath) - fHeader.Method = zip.Deflate - f, err := w.CreateHeader(fHeader) - if err != nil { - return err - } - _, err = f.Write(b) - return err - }); err != nil { - panic(err) - } - if err := w.Close(); err != nil { + bs, err := ziptree.Zip(srcPath) + if err != nil { panic(err) } - return string(out.Bytes()) + return string(bs) } // mustReadFile returns the file contents. Panics on any errors. diff --git a/statik.go b/statik.go index 91b2a64..d5a0b9e 100644 --- a/statik.go +++ b/statik.go @@ -17,7 +17,6 @@ package main import ( - "archive/zip" "bytes" "flag" "fmt" @@ -25,9 +24,10 @@ import ( "io/ioutil" "os" "path" - "path/filepath" "strings" "time" + + "github.com/rakyll/statik/ziptree" ) const ( @@ -119,65 +119,16 @@ func rename(src, dest string) error { // Generates source registers generated zip contents data to // be read by the statik/fs HTTP file system. func generateSource(srcPath string) (file *os.File, err error) { - var ( - buffer bytes.Buffer - zipWriter io.Writer - ) - - zipWriter = &buffer - f, err := ioutil.TempFile("", namePackage) - if err != nil { - return + var opts []ziptree.Option + if *flagNoCompress { + opts = append(opts, ziptree.Compress(false)) } - - zipWriter = io.MultiWriter(zipWriter, f) - defer f.Close() - - w := zip.NewWriter(zipWriter) - if err = filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - // Ignore directories and hidden files. - // No entry is needed for directories in a zip file. - // Each file is represented with a path, no directory - // entities are required to build the hierarchy. - if fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { - return nil - } - relPath, err := filepath.Rel(srcPath, path) - if err != nil { - return err - } - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - fHeader, err := zip.FileInfoHeader(fi) - if err != nil { - return err - } - if *flagNoMtime { - // Always use the same modification time so that - // the output is deterministic with respect to the file contents. - // Do NOT use fHeader.Modified as it only works on go >= 1.10 - fHeader.SetModTime(mtimeDate) - } - fHeader.Name = filepath.ToSlash(relPath) - if !*flagNoCompress { - fHeader.Method = zip.Deflate - } - f, err := w.CreateHeader(fHeader) - if err != nil { - return err - } - _, err = f.Write(b) - return err - }); err != nil { - return + if *flagNoMtime { + opts = append(opts, ziptree.FixMtime(mtimeDate)) } - if err = w.Close(); err != nil { - return + bs, err := ziptree.Zip(srcPath, opts...) + if err != nil { + return nil, err } var tags string @@ -202,12 +153,16 @@ import ( func init() { data := "`, tags, comment, namePackage) - FprintZipData(&qb, buffer.Bytes()) + FprintZipData(&qb, bs) fmt.Fprint(&qb, `" fs.Register(data) } `) - + f, err := ioutil.TempFile("", namePackage) + if err != nil { + return nil, err + } + f.Close() if err = ioutil.WriteFile(f.Name(), qb.Bytes(), 0644); err != nil { return } diff --git a/ziptree/zipper.go b/ziptree/zipper.go new file mode 100644 index 0000000..8c37f76 --- /dev/null +++ b/ziptree/zipper.go @@ -0,0 +1,90 @@ +package ziptree + +import ( + "archive/zip" + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +type Option func(*config) + +type config struct { + Compress bool + Mtime time.Time +} + +func newConfig() *config { + return &config{Compress: true} +} + +func Compress(f bool) Option { + return func(c *config) { + c.Compress = f + } +} + +func FixMtime(t time.Time) Option { + return func(c *config) { + c.Mtime = t + } +} + +func Zip(srcPath string, opts ...Option) ([]byte, error) { + c := newConfig() + for _, opt := range opts { + opt(c) + } + + var buffer bytes.Buffer + w := zip.NewWriter(&buffer) + if err := filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + // Ignore directories and hidden files. + // No entry is needed for directories in a zip file. + // Each file is represented with a path, no directory + // entities are required to build the hierarchy. + if fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { + return nil + } + relPath, err := filepath.Rel(srcPath, path) + if err != nil { + return err + } + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + fHeader, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + if !c.Mtime.IsZero() { + // Always use the same modification time so that + // the output is deterministic with respect to the file contents. + // Do NOT use fHeader.Modified as it only works on go >= 1.10 + fHeader.SetModTime(c.Mtime) + } + fHeader.Name = filepath.ToSlash(relPath) + if c.Compress { + fHeader.Method = zip.Deflate + } + f, err := w.CreateHeader(fHeader) + if err != nil { + return err + } + _, err = f.Write(b) + return err + }); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return buffer.Bytes(), nil +} From f8774031491984d6b6751b891f63cbe11b958abf Mon Sep 17 00:00:00 2001 From: Songmu Date: Sat, 23 Mar 2019 16:47:35 +0900 Subject: [PATCH 2/4] move func FprintZipData to ziptree package --- statik.go | 25 +------------------------ ziptree/fprint.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 ziptree/fprint.go diff --git a/statik.go b/statik.go index d5a0b9e..ffdc7a2 100644 --- a/statik.go +++ b/statik.go @@ -153,7 +153,7 @@ import ( func init() { data := "`, tags, comment, namePackage) - FprintZipData(&qb, bs) + ziptree.FprintZipData(&qb, bs) fmt.Fprint(&qb, `" fs.Register(data) } @@ -169,29 +169,6 @@ func init() { return f, nil } -// FprintZipData converts zip binary contents to a string literal. -func FprintZipData(dest *bytes.Buffer, zipData []byte) { - for _, b := range zipData { - if b == '\n' { - dest.WriteString(`\n`) - continue - } - if b == '\\' { - dest.WriteString(`\\`) - continue - } - if b == '"' { - dest.WriteString(`\"`) - continue - } - if (b >= 32 && b <= 126) || b == '\t' { - dest.WriteByte(b) - continue - } - fmt.Fprintf(dest, "\\x%02x", b) - } -} - // comment lines prefixes each line in lines with "// ". func commentLines(lines string) string { lines = "// " + strings.Replace(lines, "\n", "\n// ", -1) diff --git a/ziptree/fprint.go b/ziptree/fprint.go new file mode 100644 index 0000000..b109f4a --- /dev/null +++ b/ziptree/fprint.go @@ -0,0 +1,33 @@ +package ziptree + +import ( + "bufio" + "fmt" + "io" +) + +// FprintZipData converts zip binary contents to a string literal. +func FprintZipData(w io.Writer, zipData []byte) error { + dest := bufio.NewWriter(w) + for _, b := range zipData { + if err := func(b byte) error { + switch b { + case '\n': + _, err := dest.WriteString(`\n`) + return err + case '\\', '"': + _, err := dest.WriteString(`\` + string(b)) + return err + default: + if (b >= 32 && b <= 126) || b == '\t' { + return dest.WriteByte(b) + } + _, err := fmt.Fprintf(dest, "\\x%02x", b) + return err + } + }(b); err != nil { + return err + } + } + return dest.Flush() +} From c7bcca18fd6285bb28ff829458c05a653e83bf86 Mon Sep 17 00:00:00 2001 From: Songmu Date: Sat, 23 Mar 2019 20:49:55 +0900 Subject: [PATCH 3/4] add comment for ziptree package --- ziptree/zipper.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ziptree/zipper.go b/ziptree/zipper.go index 8c37f76..0932297 100644 --- a/ziptree/zipper.go +++ b/ziptree/zipper.go @@ -1,3 +1,4 @@ +// Package ziptree contains code to zip a directory tree and write it out. package ziptree import ( @@ -10,6 +11,7 @@ import ( "time" ) +// Option for zipping type Option func(*config) type config struct { @@ -21,18 +23,21 @@ func newConfig() *config { return &config{Compress: true} } +// Compress is an Option to compress a zip or not func Compress(f bool) Option { return func(c *config) { c.Compress = f } } +// FixMtime is an Option to fix mtimes of the file in the zip func FixMtime(t time.Time) Option { return func(c *config) { c.Mtime = t } } +// Zip a directory tree func Zip(srcPath string, opts ...Option) ([]byte, error) { c := newConfig() for _, opt := range opts { From e909acff05aa6cab7f228c488b1e4bed53ff69ba Mon Sep 17 00:00:00 2001 From: Songmu Date: Sat, 23 Mar 2019 21:26:02 +0900 Subject: [PATCH 4/4] add ziptree/zipper_test.go --- .travis.yml | 4 +-- testdata/ziptree/hello | 0 ziptree/zipper_test.go | 63 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 testdata/ziptree/hello create mode 100644 ziptree/zipper_test.go diff --git a/.travis.yml b/.travis.yml index 7859259..39d2ac4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.9.x - - 1.10.3 + - 1.11.x + - 1.12.1 install: - go build -v diff --git a/testdata/ziptree/hello b/testdata/ziptree/hello new file mode 100644 index 0000000..e69de29 diff --git a/ziptree/zipper_test.go b/ziptree/zipper_test.go new file mode 100644 index 0000000..15dc6c2 --- /dev/null +++ b/ziptree/zipper_test.go @@ -0,0 +1,63 @@ +package ziptree_test + +import ( + "bytes" + "os" + "reflect" + "testing" + "time" + + "github.com/rakyll/statik/ziptree" +) + +var zipData = []byte{ + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x21, 0x28, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x05, 0x00, 0x09, 0x00, 0x68, 0x65, + 0x6c, 0x6c, 0x6f, 0x55, 0x54, 0x05, 0x00, 0x01, + 0x80, 0x43, 0x6d, 0x38, 0x01, 0x00, 0x00, 0xff, + 0xff, 0x50, 0x4b, 0x07, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, + 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x21, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa4, 0x81, 0x00, 0x00, 0x00, 0x00, 0x68, + 0x65, 0x6c, 0x6c, 0x6f, 0x55, 0x54, 0x05, 0x00, + 0x01, 0x80, 0x43, 0x6d, 0x38, 0x50, 0x4b, 0x05, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x3c, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, + 0x00, 0x00, 0x00} +var zipData4Print = zipData + +func TestZip(t *testing.T) { + err := os.Chmod("../testdata/ziptree/hello", 0644) + if err != nil { + t.Fatal(err) + } + out, err := ziptree.Zip( + "../testdata/ziptree/", + ziptree.FixMtime(time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC))) + if err != nil { + t.Errorf("error should be nil but: %s", err) + } + wantData := zipData + if !reflect.DeepEqual(out, wantData) { + t.Errorf("got: %#v\nexpect: %#v", out, wantData) + } +} + +func TestFprintZipData(t *testing.T) { + buf := &bytes.Buffer{} + err := ziptree.FprintZipData(buf, zipData4Print) + if err != nil { + t.Errorf("error should be nil but: %s", err) + } + out := buf.String() + want := `PK\x03\x04\x14\x00\x08\x00\x08\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00 \x00helloUT\x05\x00\x01\x80Cm8\x01\x00\x00\xff\xffPK\x07\x08\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x08\x00\x00\x00!(\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x05\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00helloUT\x05\x00\x01\x80Cm8PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00<\x00\x00\x00A\x00\x00\x00\x00\x00` + if out != want { + t.Errorf("got: %s\nexpect: %s", out, want) + } +}