Skip to content
Permalink
Browse files
fix: source archive add subfolders (#3343)
`--add-file` adds only the `--prefix` joined with the basename of the added file, so, adding a folder like `vendor` would break havoc.

this basically writes our own source archives (which are more compressed) and allows to add an entire folder easily, as well as other mappings as `archives` already supports. 

refs #3102 #2911

Signed-off-by: Carlos A Becker <caarlos0@users.noreply.github.com>
  • Loading branch information
caarlos0 committed Aug 25, 2022
1 parent 12c4256 commit d7acf21f03d8e641fcf592043bf2baf934a678cc
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 73 deletions.
@@ -2,13 +2,17 @@
package sourcearchive

import (
"fmt"
"os"
"path/filepath"
"strings"

"github.com/caarlos0/log"
"github.com/goreleaser/goreleaser/internal/archivefiles"
"github.com/goreleaser/goreleaser/internal/artifact"
"github.com/goreleaser/goreleaser/internal/git"
"github.com/goreleaser/goreleaser/internal/tmpl"
"github.com/goreleaser/goreleaser/pkg/archive"
"github.com/goreleaser/goreleaser/pkg/config"
"github.com/goreleaser/goreleaser/pkg/context"
)
@@ -33,31 +37,54 @@ func (Pipe) Run(ctx *context.Context) (err error) {
filename := name + "." + ctx.Config.Source.Format
path := filepath.Join(ctx.Config.Dist, filename)
log.WithField("file", filename).Info("creating source archive")
args := []string{
"archive",
"-o", path,
"--format", ctx.Config.Source.Format,

out, err := git.Run(ctx, "ls-files")
if err != nil {
return fmt.Errorf("could not list source files: %w", err)
}

if ctx.Config.Source.PrefixTemplate != "" {
prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
if err != nil {
return err
}
args = append(args, "--prefix", prefix)
prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
if err != nil {
return err
}

files, err := evalFiles(ctx)
af, err := os.Create(path)
if err != nil {
return fmt.Errorf("could not create archive: %w", err)
}
defer af.Close() //nolint:errcheck

arch, err := archive.New(af, ctx.Config.Source.Format)
if err != nil {
return err
}

var ff []config.File
for _, f := range strings.Split(out, "\n") {
if strings.TrimSpace(f) == "" {
continue
}
ff = append(ff, config.File{
Source: f,
})
}
files, err := archivefiles.Eval(tmpl.New(ctx), append(ff, ctx.Config.Source.Files...))
if err != nil {
return err
}
for _, f := range files {
args = append(args, "--add-file", f)
f.Destination = filepath.Join(prefix, f.Destination)
if err := arch.Add(f); err != nil {
return fmt.Errorf("could not add %q to archive: %w", f.Source, err)
}
}

args = append(args, ctx.Git.FullCommit)
out, err := git.Clean(git.Run(ctx, args...))
log.Debug(out)
if err := arch.Close(); err != nil {
return fmt.Errorf("could not close archive file: %w", err)
}
if err := af.Close(); err != nil {
return fmt.Errorf("could not close archive file: %w", err)
}

ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.UploadableSourceArchive,
@@ -70,41 +97,6 @@ func (Pipe) Run(ctx *context.Context) (err error) {
return err
}

// to reuse the archivefiles packages, we do something funky:
// - convert the []string to []config.File
// - eval it in archivefiles
// - convert it back to []string
//
// we also handle files already tracked, as if we add them again,
// they'll get duplicated in the archive.
func evalFiles(ctx *context.Context) ([]string, error) {
var files []config.File
for _, f := range ctx.Config.Source.Files {
files = append(files, config.File{
Source: f,
})
}
addFiles, err := archivefiles.Eval(tmpl.New(ctx), files)
if err != nil {
return nil, err
}

var result []string
for _, f := range addFiles {
if isTracked(ctx, f.Source) {
continue
}
result = append(result, f.Source)
}
return result, nil
}

// check if file is tracked, and, if it is we should not add it to the archive again.
func isTracked(ctx *context.Context, path string) bool {
_, err := git.Run(ctx, "ls-files", "--error-unmatch", path)
return err == nil
}

// Default sets the pipe defaults.
func (Pipe) Default(ctx *context.Context) error {
archive := &ctx.Config.Source
@@ -27,6 +27,8 @@ func TestArchive(t *testing.T) {
testlib.GitCommit(t, "feat: first")
require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655))
require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655))
require.NoError(t, os.MkdirAll("subfolder", 0o755))
require.NoError(t, os.WriteFile("subfolder/file.md", []byte("a file within a folder, added later"), 0o655))

ctx := context.New(config.Project{
ProjectName: "foo",
@@ -35,8 +37,9 @@ func TestArchive(t *testing.T) {
Format: format,
Enabled: true,
PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/",
Files: []string{
"*.txt",
Files: []config.File{
{Source: "*.txt"},
{Source: "subfolder/*"},
},
},
})
@@ -65,22 +68,13 @@ func TestArchive(t *testing.T) {
return
}

f, err := os.Open(path)
require.NoError(t, err)
z, err := zip.NewReader(f, stat.Size())
require.NoError(t, err)

var paths []string
for _, zf := range z.File {
paths = append(paths, zf.Name)
}
require.Equal(t, []string{
"foo-1.0.0/",
require.ElementsMatch(t, []string{
"foo-1.0.0/README.md",
"foo-1.0.0/code.py",
"foo-1.0.0/code.txt",
"foo-1.0.0/added-later.txt",
}, paths)
"foo-1.0.0/subfolder/file.md",
}, lsZip(t, path))
})
}
}
@@ -96,7 +90,7 @@ func TestInvalidFormat(t *testing.T) {
},
})
require.NoError(t, Pipe{}.Default(ctx))
require.EqualError(t, Pipe{}.Run(ctx), "fatal: Unknown archive format '7z'")
require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z")
}

func TestDefault(t *testing.T) {
@@ -115,7 +109,45 @@ func TestInvalidNameTemplate(t *testing.T) {
NameTemplate: "{{ .foo }-asdda",
},
})
require.EqualError(t, Pipe{}.Run(ctx), "template: tmpl:1: unexpected \"}\" in operand")
testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
}

func TestInvalidInvalidFileTemplate(t *testing.T) {
testlib.Mktmp(t)
require.NoError(t, os.Mkdir("dist", 0o744))

testlib.GitInit(t)
require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655))
testlib.GitAdd(t)
testlib.GitCommit(t, "feat: first")

ctx := context.New(config.Project{
ProjectName: "foo",
Dist: "dist",
Source: config.Source{
Format: "tar.gz",
Enabled: true,
Files: []config.File{
{Source: "{{.Test}"},
},
},
})
ctx.Git.FullCommit = "HEAD"
ctx.Version = "1.0.0"
require.NoError(t, Pipe{}.Default(ctx))
testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
}

func TestInvalidPrefixTemplate(t *testing.T) {
ctx := context.New(config.Project{
Dist: t.TempDir(),
Source: config.Source{
Enabled: true,
PrefixTemplate: "{{ .ProjectName }/",
},
})
require.NoError(t, Pipe{}.Default(ctx))
testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
}

func TestDisabled(t *testing.T) {
@@ -136,3 +168,24 @@ func TestSkip(t *testing.T) {
require.False(t, Pipe{}.Skip(ctx))
})
}

func TestString(t *testing.T) {
require.NotEmpty(t, Pipe{}.String())
}

func lsZip(tb testing.TB, path string) []string {
tb.Helper()

stat, err := os.Stat(path)
require.NoError(tb, err)
f, err := os.Open(path)
require.NoError(tb, err)
z, err := zip.NewReader(f, stat.Size())
require.NoError(tb, err)

var paths []string
for _, zf := range z.File {
paths = append(paths, zf.Name)
}
return paths
}
@@ -876,11 +876,11 @@ type Publisher struct {

// Source configuration.
type Source struct {
NameTemplate string `yaml:"name_template,omitempty" json:"name_template,omitempty"`
Format string `yaml:"format,omitempty" json:"format,omitempty"`
Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"`
Files []string `yaml:"files,omitempty" json:"files,omitempty"`
NameTemplate string `yaml:"name_template,omitempty" json:"name_template,omitempty"`
Format string `yaml:"format,omitempty" json:"format,omitempty"`
Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"`
PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"`
Files []File `yaml:"files,omitempty" json:"files,omitempty"`
}

// Project includes all project configuration.
@@ -25,7 +25,6 @@ source:
prefix_template: '{{ .ProjectName }}-{{ .Version }}/'
# Additional files/template/globs you want to add to the source archive.
# Will use --add-file of git-archive.
# Defaults to empty.
files:
- LICENSE.txt
@@ -34,6 +33,22 @@ source:
- docs/*
- design/*.png
- templates/**/*
# a more complete example, check the globbing deep dive below
- src: '*.md'
dst: docs
# Strip parent folders when adding files to the archive.
# Default: false
strip_parent: true
# File info.
# Not all fields are supported by all formats available formats.
# Defaults to the file info of the actual file if not provided.
info:
owner: root
group: root
mode: 0644
# format is `time.RFC3339Nano`
mtime: 2008-01-02T15:04:05Z
```

!!! tip

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit d7acf21

Please sign in to comment.