Skip to content

Commit

Permalink
Add support for zip archives to .chezmoiexternal
Browse files Browse the repository at this point in the history
  • Loading branch information
twpayne committed Sep 1, 2021
1 parent 5e6f7e6 commit ad1c938
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 47 deletions.
3 changes: 2 additions & 1 deletion docs/REFERENCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,8 @@ archive at `url`. The optional boolean field `exact` may be set, in which case
the directory and all subdirectories will be treated as exact directories, i.e.
`chezmoi apply` will remove entries not present in the archive. The optional
integer field `stripComponents` will remove leading path components from the
members of archive.
members of archive. The supported archive formats are `.tar`, `.tar.gz`, `.tgz`,
`.tar.bz2`, `.tbz2`, and `.zip`.

By default, chezmoi will cache downloaded URLs the first time they are accessed.
To force chezmoi to re-download URLs, pass the `--refresh-externals` flag.
Expand Down
96 changes: 96 additions & 0 deletions internal/chezmoi/archive.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package chezmoi

import (
"archive/tar"
"archive/zip"
"bytes"
"compress/bzip2"
"compress/gzip"
"errors"
"fmt"
"io"
"io/fs"
"path"
"strings"
)

var errUnknownFormat = errors.New("unknown format")

// An walkArchiveFunc is called once for each entry in an archive.
type walkArchiveFunc func(name string, info fs.FileInfo, r io.Reader, linkname string) error

// walkArchive walks over all the entries in an archive. path is used as a hint
// for the archive format.
func walkArchive(path string, data []byte, f walkArchiveFunc) error {
pathLower := strings.ToLower(path)
if strings.HasSuffix(pathLower, ".zip") {
return walkArchiveZip(bytes.NewReader(data), int64(len(data)), f)
}
var r io.Reader = bytes.NewReader(data)
switch {
case strings.HasSuffix(pathLower, ".tar"):
case strings.HasSuffix(pathLower, ".tar.bz2") || strings.HasSuffix(pathLower, ".tbz2"):
r = bzip2.NewReader(r)
case strings.HasSuffix(pathLower, ".tar.gz") || strings.HasSuffix(pathLower, ".tgz"):
var err error
r, err = gzip.NewReader(r)
if err != nil {
return err
}
default:
return errUnknownFormat
}
return walkArchiveTar(r, f)
}

// walkArchiveTar walks over all the entries in a tar archive.
func walkArchiveTar(r io.Reader, f walkArchiveFunc) error {
tarReader := tar.NewReader(r)
for {
header, err := tarReader.Next()
switch {
case errors.Is(err, io.EOF):
return nil
case err != nil:
return err
}
name := strings.TrimSuffix(header.Name, "/")
switch header.Typeflag {
case tar.TypeDir, tar.TypeReg:
if err := f(name, header.FileInfo(), tarReader, ""); err != nil {
return err
}
case tar.TypeSymlink:
if err := f(name, header.FileInfo(), nil, header.Linkname); err != nil {
return err
}
case tar.TypeXGlobalHeader:
default:
return fmt.Errorf("%s: unsupported typeflag '%c'", header.Name, header.Typeflag)
}
}
}

// walkArchiveZip walks over all the entries in a zip archive.
func walkArchiveZip(r io.ReaderAt, size int64, f walkArchiveFunc) error {
zipReader, err := zip.NewReader(r, size)
if err != nil {
return err
}
for _, zipFile := range zipReader.File {
zipFileReader, err := zipFile.Open()
if err != nil {
return err
}
name := path.Clean(zipFile.Name)
if strings.HasPrefix(name, "../") {
return fmt.Errorf("%s: invalid filename", zipFile.Name)
}
err = f(name, zipFile.FileInfo(), zipFileReader, "")
zipFileReader.Close()
if err != nil {
return err
}
}
return nil
}
62 changes: 20 additions & 42 deletions internal/chezmoi/sourcestate.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,8 @@ package chezmoi
// FIXME implement include and exclude entry type sets for externals

import (
"archive/tar"
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"encoding/hex"
"errors"
Expand Down Expand Up @@ -1481,24 +1478,10 @@ func (s *SourceState) readExternalArchive(ctx context.Context, externalRelPath R
if err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
}
var r io.Reader = bytes.NewReader(data)
path := url.Path
if external.Encrypted {
path = strings.TrimSuffix(path, s.encryption.EncryptedSuffix())
}
switch {
case strings.HasSuffix(path, ".tar.gz") || strings.HasSuffix(path, ".tgz"):
r, err = gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
}
case strings.HasSuffix(path, ".tar.bz2") || strings.HasSuffix(path, ".tbz2"):
r = bzip2.NewReader(r)
default:
return nil, fmt.Errorf("%s: %s: unknown format", externalRelPath, external.URL)
}

tarReader := tar.NewReader(r)
sourceRelPath := NewSourceRelPath(RelPath(external.URL))
sourceStateEntries := map[RelPath][]SourceStateEntry{
externalRelPath: {
Expand All @@ -1513,35 +1496,29 @@ func (s *SourceState) readExternalArchive(ctx context.Context, externalRelPath R
},
},
}
FOR:
for {
header, err := tarReader.Next()
switch {
case errors.Is(err, io.EOF):
break FOR
case err != nil:
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
}

name := strings.TrimSuffix(header.Name, "/")
if err := walkArchive(path, data, func(name string, info fs.FileInfo, r io.Reader, linkname string) error {
if external.StripComponents > 0 {
components := strings.Split(name, "/")
if len(components) <= external.StripComponents {
continue FOR
return nil
}
name = strings.Join(components[external.StripComponents:], "/")
}
if name == "" {
return nil
}
targetRelPath := externalRelPath.Join(RelPath(name))

if s.Ignored(targetRelPath) {
continue FOR
return nil
}

var sourceStateEntry SourceStateEntry
switch header.Typeflag {
case tar.TypeDir:
switch {
case info.IsDir():
targetStateEntry := &TargetStateDir{
perm: fs.FileMode(header.Mode).Perm() &^ s.umask,
perm: info.Mode().Perm() &^ s.umask,
}
sourceStateEntry = &SourceStateDir{
Attr: DirAttr{
Expand All @@ -1550,15 +1527,15 @@ FOR:
sourceRelPath: sourceRelPath,
targetStateEntry: targetStateEntry,
}
case tar.TypeReg:
contents, err := io.ReadAll(tarReader)
case info.Mode()&fs.ModeType == 0:
contents, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("%s: %s: %s: %w", externalRelPath, external.URL, header.Name, err)
return fmt.Errorf("%s: %w", name, err)
}
lazyContents := newLazyContents(contents)
fileAttr := FileAttr{
Empty: true,
Executable: header.FileInfo().Mode()&0o111 != 0,
Executable: info.Mode().Perm()&0o111 != 0,
}
targetStateEntry := &TargetStateFile{
lazyContents: lazyContents,
Expand All @@ -1570,22 +1547,23 @@ FOR:
sourceRelPath: sourceRelPath,
targetStateEntry: targetStateEntry,
}
case tar.TypeSymlink:
case info.Mode()&fs.ModeType == fs.ModeSymlink:
targetStateEntry := &TargetStateSymlink{
lazyLinkname: newLazyLinkname(header.Linkname),
lazyLinkname: newLazyLinkname(linkname),
}
sourceStateEntry = &SourceStateFile{
sourceRelPath: sourceRelPath,
targetStateEntry: targetStateEntry,
}
case tar.TypeXGlobalHeader:
continue FOR
default:
return nil, fmt.Errorf("%s: %s: unsupported typeflag: '%c'", externalRelPath, external.URL, header.Typeflag)
return fmt.Errorf("%s: unsupported mode %o", name, info.Mode()&fs.ModeType)
}

sourceStateEntries[targetRelPath] = append(sourceStateEntries[targetRelPath], sourceStateEntry)
return nil
}); err != nil {
return nil, fmt.Errorf("%s: %s: %w", externalRelPath, external.URL, err)
}

return sourceStateEntries, nil
}

Expand Down
8 changes: 4 additions & 4 deletions internal/cmd/importcmd.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package cmd

// LATER add zip import
// LATER add zip import using chezmoi.walkArchive

import (
"archive/tar"
Expand Down Expand Up @@ -66,14 +66,14 @@ func (c *Config) runImportCmd(cmd *cobra.Command, args []string, sourceState *ch
}
r = bytes.NewReader(data)
switch base := strings.ToLower(absPath.Base()); {
case strings.HasSuffix(base, ".tar"):
case strings.HasSuffix(base, ".tar.bz2") || strings.HasSuffix(base, ".tbz2"):
r = bzip2.NewReader(r)
case strings.HasSuffix(base, ".tar.gz") || strings.HasSuffix(base, ".tgz"):
r, err = gzip.NewReader(r)
if err != nil {
return err
}
case strings.HasSuffix(base, ".tar.bz2") || strings.HasSuffix(base, ".tbz2"):
r = bzip2.NewReader(r)
case strings.HasSuffix(base, ".tar"):
default:
return fmt.Errorf("unknown format: %s", base)
}
Expand Down
22 changes: 22 additions & 0 deletions internal/cmd/testdata/scripts/externalzip.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[!exec:zip] skip 'zip not found in $PATH'
exec zip -r www/archive.zip archive

httpd www

# test that chezmoi reads external zip archives from .chezmoiexternal.json
chezmoi apply --force
cmp $HOME/.dir/dir/file golden/dir/file

-- archive/dir/file --
# contents of dir/file
-- golden/dir/file --
# contents of dir/file
-- home/user/.local/share/chezmoi/.chezmoiexternal.json --
{
".dir": {
"type": "archive",
"url": "{{ env "HTTPD_URL" }}/archive.zip",
"stripComponents": 1
}
}
-- www/.keep --

0 comments on commit ad1c938

Please sign in to comment.