diff --git a/cmd/input.go b/cmd/input.go index 36af6d866cd..59c14002b2c 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -59,6 +59,7 @@ type Input struct { logPrefixJobID bool networkName string useNewActionCache bool + localRepository []string } func (i *Input) resolve(path string) string { diff --git a/cmd/root.go b/cmd/root.go index e506699fd5e..bb57ec9d955 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -100,6 +100,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().BoolVarP(&input.actionOfflineMode, "action-offline-mode", "", false, "If action contents exists, it will not be fetch and pull again. If turn on this,will turn off force pull") rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.") rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally") + rootCmd.PersistentFlags().StringArrayVarP(&input.localRepository, "local-repository", "", []string{}, "Replaces the specified repository and ref with a local folder") rootCmd.SetArgs(args()) if err := rootCmd.Execute(); err != nil { @@ -618,10 +619,22 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str Matrix: matrixes, ContainerNetworkMode: docker_container.NetworkMode(input.networkName), } - if input.useNewActionCache { + if input.useNewActionCache || len(input.localRepository) > 0 { config.ActionCache = &runner.GoGitActionCache{ Path: config.ActionCacheDir, } + if len(input.localRepository) > 0 { + localRepositories := map[string]string{} + for _, l := range input.localRepository { + k, v, _ := strings.Cut(l, "=") + localRepositories[k] = v + } + config.ActionCache = &runner.LocalRepositoryCache{ + Parent: config.ActionCache, + LocalRepositories: localRepositories, + CacheDirCache: map[string]string{}, + } + } } r, err := runner.New(config) if err != nil { diff --git a/pkg/runner/file_collector.go b/pkg/runner/file_collector.go new file mode 100644 index 00000000000..45a13683abf --- /dev/null +++ b/pkg/runner/file_collector.go @@ -0,0 +1,187 @@ +package runner + +import ( + "archive/tar" + "context" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + git "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/go-git/go-git/v5/plumbing/format/index" +) + +type fileCollectorHandler interface { + WriteFile(path string, fi fs.FileInfo, linkName string, f io.Reader) error +} + +type tarCollector struct { + TarWriter *tar.Writer + UID int + GID int + DstDir string +} + +func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { + // create a new dir/file header + header, err := tar.FileInfoHeader(fi, linkName) + if err != nil { + return err + } + + // update the name to correctly reflect the desired destination when untaring + header.Name = path.Join(tc.DstDir, fpath) + header.Mode = int64(fi.Mode()) + header.ModTime = fi.ModTime() + header.Uid = tc.UID + header.Gid = tc.GID + + // write the header + if err := tc.TarWriter.WriteHeader(header); err != nil { + return err + } + + // this is a symlink no reader provided + if f == nil { + return nil + } + + // copy file data into tar writer + if _, err := io.Copy(tc.TarWriter, f); err != nil { + return err + } + return nil +} + +type fileCollector struct { + Ignorer gitignore.Matcher + SrcPath string + SrcPrefix string + Fs fileCollectorFs + Handler fileCollectorHandler +} + +type fileCollectorFs interface { + Walk(root string, fn filepath.WalkFunc) error + OpenGitIndex(path string) (*index.Index, error) + Open(path string) (io.ReadCloser, error) + Readlink(path string) (string, error) +} + +type defaultFs struct { +} + +func (*defaultFs) Walk(root string, fn filepath.WalkFunc) error { + return filepath.Walk(root, fn) +} + +func (*defaultFs) OpenGitIndex(path string) (*index.Index, error) { + r, err := git.PlainOpen(path) + if err != nil { + return nil, err + } + i, err := r.Storer.Index() + if err != nil { + return nil, err + } + return i, nil +} + +func (*defaultFs) Open(path string) (io.ReadCloser, error) { + return os.Open(path) +} + +func (*defaultFs) Readlink(path string) (string, error) { + return os.Readlink(path) +} + +//nolint:gocyclo +func (fc *fileCollector) collectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc { + i, _ := fc.Fs.OpenGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...))) + return func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if ctx != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("copy cancelled") + default: + } + } + + sansPrefix := strings.TrimPrefix(file, fc.SrcPrefix) + split := strings.Split(sansPrefix, string(filepath.Separator)) + // The root folders should be skipped, submodules only have the last path component set to "." by filepath.Walk + if fi.IsDir() && len(split) > 0 && split[len(split)-1] == "." { + return nil + } + var entry *index.Entry + if i != nil { + entry, err = i.Entry(strings.Join(split[len(submodulePath):], "/")) + } else { + err = index.ErrEntryNotFound + } + if err != nil && fc.Ignorer != nil && fc.Ignorer.Match(split, fi.IsDir()) { + if fi.IsDir() { + if i != nil { + ms, err := i.Glob(strings.Join(append(split[len(submodulePath):], "**"), "/")) + if err != nil || len(ms) == 0 { + return filepath.SkipDir + } + } else { + return filepath.SkipDir + } + } else { + return nil + } + } + if err == nil && entry.Mode == filemode.Submodule { + err = fc.Fs.Walk(file, fc.collectFiles(ctx, split)) + if err != nil { + return err + } + return filepath.SkipDir + } + path := filepath.ToSlash(sansPrefix) + + // return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update) + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + linkName, err := fc.Fs.Readlink(file) + if err != nil { + return fmt.Errorf("unable to readlink '%s': %w", file, err) + } + return fc.Handler.WriteFile(path, fi, linkName, nil) + } else if !fi.Mode().IsRegular() { + return nil + } + + // open file + f, err := fc.Fs.Open(file) + if err != nil { + return err + } + defer f.Close() + + if ctx != nil { + // make io.Copy cancellable by closing the file + cpctx, cpfinish := context.WithCancel(ctx) + defer cpfinish() + go func() { + select { + case <-cpctx.Done(): + case <-ctx.Done(): + f.Close() + } + }() + } + + return fc.Handler.WriteFile(path, fi, "", f) + } +} diff --git a/pkg/runner/local_repository_cache.go b/pkg/runner/local_repository_cache.go new file mode 100644 index 00000000000..0cfe83a56f4 --- /dev/null +++ b/pkg/runner/local_repository_cache.go @@ -0,0 +1,81 @@ +package runner + +import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +type LocalRepositoryCache struct { + Parent ActionCache + LocalRepositories map[string]string + CacheDirCache map[string]string +} + +func (l *LocalRepositoryCache) Fetch(ctx context.Context, cacheDir, url, ref, token string) (string, error) { + if dest, ok := l.LocalRepositories[fmt.Sprintf("%s@%s", url, ref)]; ok { + l.CacheDirCache[cacheDir] = dest + return "local-repository", nil + } + return l.Parent.Fetch(ctx, cacheDir, url, ref, token) +} + +func (l *LocalRepositoryCache) GetTarArchive(ctx context.Context, cacheDir, sha, includePrefix string) (io.ReadCloser, error) { + if dest, ok := l.CacheDirCache[cacheDir]; ok { + srcPath := filepath.Join(dest, includePrefix) + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer tw.Close() + srcPath = filepath.Clean(srcPath) + fi, err := os.Lstat(srcPath) + if err != nil { + return nil, err + } + tc := &tarCollector{ + TarWriter: tw, + } + if fi.IsDir() { + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: tc, + } + err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + if err != nil { + return nil, err + } + } else { + var f io.ReadCloser + var linkname string + if fi.Mode()&fs.ModeSymlink != 0 { + linkname, err = os.Readlink(srcPath) + if err != nil { + return nil, err + } + } else { + f, err = os.Open(srcPath) + if err != nil { + return nil, err + } + defer f.Close() + } + err := tc.WriteFile(fi.Name(), fi, linkname, f) + if err != nil { + return nil, err + } + } + return io.NopCloser(buf), nil + } + return l.Parent.GetTarArchive(ctx, cacheDir, sha, includePrefix) +}