feat: Add refreshPeriod option to externals

docs/HOWTO.md

@@ -361,21 +361,29 @@ chezmoi uses its own format for the source state and Oh My Zsh is not
 distributed in this format. Instead, you can use the `.chezmoiexternal.<format>`
 to tell chezmoi to import dotfiles from an external source.
 
-For example, to import Oh My Zsh and the [zsh-syntax-highlighting
-plugin](https://github.com/zsh-users/zsh-syntax-highlighting), put the following
-in `~/.local/share/chezmoi/.chezmoiexternal.toml`:
+For example, to import Oh My Zsh, the [zsh-syntax-highlighting
+plugin](https://github.com/zsh-users/zsh-syntax-highlighting), and
+[powerlevel10k](https://github.com/romkatv/powerlevel10k), put the following in
+`~/.local/share/chezmoi/.chezmoiexternal.toml`:
 
 ```toml
 [".oh-my-zsh"]
     type = "archive"
     url = "https://github.com/ohmyzsh/ohmyzsh/archive/master.tar.gz"
     exact = true
     stripComponents = 1
+    refreshPeriod = "1w"
 [".oh-my-zsh/custom/plugins/zsh-syntax-highlighting"]
     type = "archive"
     url = "https://github.com/zsh-users/zsh-syntax-highlighting/archive/master.tar.gz"
     exact = true
     stripComponents = 1
+    refreshPeriod = "1w"
+[".oh-my-zsh/custom/themes/powerlevel10k"]
+    type = "archive"
+    url = "https://github.com/romkatv/powerlevel10k/archive/v1.15.0.tar.gz"
+    exact = true
+    stripComponents = 1
 ```
 
 To apply the changes, run:
@@ -386,8 +394,18 @@ $ chezmoi apply
 
 chezmoi will download the archives and unpack them as if they were part of the
 source state. chezmoi caches downloaded archives locally to avoid re-downloading
-them every time you run a chezmoi command. To refresh the downloaded archives,
-use the `--refresh-externals` flag to `chezmoi apply`:
+them every time you run a chezmoi command, and will only re-download them at
+most every `refreshPeriod` (default never).
+
+In the above example `refreshPeriod` is set to `1w` (one week) for `.oh-my-zsh`
+and `.oh-my-zsh/custom/plugins/zsh-syntax-highlighting` because the URL point to
+tarballs of the `master` branch, which changes over time. No refresh period is
+set for `.oh-my-zsh/custom/themes/powerlevel10k` because the URL points to the a
+tarball of a tagged version, which does not change over time. To bump the
+version of powerlevel10k, change the version in the URL.
+
+To force a refresh the downloaded archives, use the `--refresh-externals` flag
+to `chezmoi apply`:
 
 ```console
 $ chezmoi --refresh-externals apply

docs/REFERENCE.md

@@ -721,25 +721,37 @@ The supported archive formats are `tar`, `tar.gz`, `tgz`, `tar.bz2`, `tbz2`, and
 `zip`. If `format` is not specified then chezmoi will guess the format using
 firstly the path of the URL and secondly its contents.
 
-By default, chezmoi will cache downloaded URLs the first time they are accessed.
-To force chezmoi to re-download URLs, pass the `--refresh-externals` flag.
+By default, chezmoi will cache downloaded URLs. The optional duration
+`refreshPeriod` field specifies how often chezmoi will re-download the URL. The
+default is zero meaning that chezmoi will never re-download unless forced. To
+force chezmoi to re-download URLs, pass the `-R`/`--refresh-externals` flag.
+Suitable refresh periods include one day (`1d`), one week (`1w`), or four weeks
+(`4w`).
 
 #### `.chezmoiexternal.<format>` examples
 
 ```toml
 [".vim/autoload/plug.vim"]
     type = "file"
     url = "https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim"
+    refreshPeriod = "1w"
 [".oh-my-zsh"]
     type = "archive"
     url = "https://github.com/ohmyzsh/ohmyzsh/archive/master.tar.gz"
     exact = true
     stripComponents = 1
+    refreshPeriod = "1w"
 [".oh-my-zsh/custom/plugins/zsh-syntax-highlighting"]
     type = "archive"
     url = "https://github.com/zsh-users/zsh-syntax-highlighting/archive/master.tar.gz"
     exact = true
     stripComponents = 1
+    refreshPeriod = "1w"
+[".oh-my-zsh/custom/themes/powerlevel10k"]
+    type = "archive"
+    url = "https://github.com/romkatv/powerlevel10k/archive/v1.15.0.tar.gz"
+    exact = true
+    stripComponents = 1
 ```
 
 ---

internal/chezmoi/sourcestate.go

@@ -21,6 +21,7 @@ import (
 	"sort"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/coreos/go-semver/semver"
 	"github.com/rs/zerolog/log"
@@ -48,10 +49,20 @@ type External struct {
 		Args    []string `json:"args" toml:"args" yaml:"args"`
 	} `json:"filter" toml:"filter" yaml:"filter"`
 	Format          ArchiveFormat `json:"format" toml:"format" yaml:"format"`
+	RefreshPeriod   time.Duration `json:"refreshPeriod" toml:"refreshPeriod" yaml:"refreshPeriod"`
 	StripComponents int           `json:"stripComponents" toml:"stripComponents" yaml:"stripComponents"`
 	URL             string        `json:"url" toml:"url" yaml:"url"`
 }
 
+// A externalCacheEntry is an external cache entry.
+type externalCacheEntry struct {
+	URL  string    `json:"url" toml:"url" time:"url"`
+	Time time.Time `json:"time" toml:"time" yaml:"time"`
+	Data []byte    `json:"data" toml:"data" yaml:"data"`
+}
+
+var externalCacheFormat = formatGzippedJSON{}
+
 // A SourceState is a source state.
 type SourceState struct {
 	entries                 map[RelPath]SourceStateEntry
@@ -643,6 +654,7 @@ func (s *SourceState) MustEntry(targetRelPath RelPath) SourceStateEntry {
 // ReadOptions are options to SourceState.Read.
 type ReadOptions struct {
 	RefreshExternals bool
+	TimeNow          func() time.Time
 }
 
 // Read reads the source state from the source directory.
@@ -1060,15 +1072,29 @@ func (s *SourceState) executeTemplate(templateAbsPath AbsPath) ([]byte, error) {
 }
 
 func (s *SourceState) getExternalDataRaw(ctx context.Context, externalRelPath RelPath, external External, options *ReadOptions) ([]byte, error) {
+	// FIXME be more intelligent about HTTP caching, e.g. following RFC 7234,
+	// rather than blindly re-downloading each time
+
+	var now time.Time
+	if options != nil && options.TimeNow != nil {
+		now = options.TimeNow()
+	} else {
+		now = time.Now()
+	}
+	now = now.UTC()
+
 	cacheKey := hex.EncodeToString(SHA256Sum([]byte(external.URL)))
-	cachedDataAbsPath := s.cacheDirAbsPath.Join("external", RelPath(cacheKey))
+	cachedDataAbsPath := s.cacheDirAbsPath.Join("external", RelPath(cacheKey+"."+externalCacheFormat.Name()))
 	if options == nil || !options.RefreshExternals {
-		data, err := s.system.ReadFile(cachedDataAbsPath)
-		switch {
-		case err == nil:
-			return data, nil
-		case !errors.Is(err, fs.ErrNotExist):
-			return nil, err
+		if data, err := s.system.ReadFile(cachedDataAbsPath); err == nil {
+			var externalCacheEntry externalCacheEntry
+			if err := externalCacheFormat.Unmarshal(data, &externalCacheEntry); err == nil {
+				if externalCacheEntry.URL == external.URL {
+					if external.RefreshPeriod == 0 || externalCacheEntry.Time.Add(external.RefreshPeriod).After(now) {
+						return externalCacheEntry.Data, nil
+					}
+				}
+			}
 		}
 	}
 
@@ -1095,10 +1121,18 @@ func (s *SourceState) getExternalDataRaw(ctx context.Context, externalRelPath Re
 		return nil, fmt.Errorf("%s: %s: %s", externalRelPath, external.URL, resp.Status)
 	}
 
+	cachedExternalData, err := externalCacheFormat.Marshal(&externalCacheEntry{
+		URL:  external.URL,
+		Time: now,
+		Data: data,
+	})
+	if err != nil {
+		return nil, err
+	}
 	if err := MkdirAll(s.baseSystem, cachedDataAbsPath.Dir(), 0o700); err != nil {
 		return nil, err
 	}
-	if err := s.baseSystem.WriteFile(cachedDataAbsPath, data, 0o600); err != nil {
+	if err := s.baseSystem.WriteFile(cachedDataAbsPath, cachedExternalData, 0o600); err != nil {
 		return nil, err
 	}
 

internal/chezmoi/sourcestate_test.go

@@ -1,6 +1,8 @@
 package chezmoi
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"errors"
 	"io/fs"
@@ -9,6 +11,7 @@ import (
 	"path/filepath"
 	"testing"
 	"text/template"
+	"time"
 
 	"github.com/coreos/go-semver/semver"
 	"github.com/stretchr/testify/assert"
@@ -1296,6 +1299,72 @@ func TestSourceStateReadExternal(t *testing.T) {
 	}
 }
 
+func TestSourceStateReadExternalCache(t *testing.T) {
+	buffer := &bytes.Buffer{}
+	tarWriterSystem := NewTARWriterSystem(buffer, tar.Header{})
+	require.NoError(t, tarWriterSystem.WriteFile(NewAbsPath("file"), []byte("# contents of file\n"), 0o666))
+	require.NoError(t, tarWriterSystem.Close())
+	archiveData := buffer.Bytes()
+
+	httpRequests := 0
+	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		httpRequests++
+		_, err := w.Write(archiveData)
+		require.NoError(t, err)
+	}))
+	defer httpServer.Close()
+
+	now := time.Now()
+	readOptions := &ReadOptions{
+		TimeNow: func() time.Time {
+			return now
+		},
+	}
+
+	chezmoitest.WithTestFS(t, map[string]interface{}{
+		"/home/user/.local/share/chezmoi": map[string]interface{}{
+			".chezmoiexternal.yaml": chezmoitest.JoinLines(
+				`.dir:`,
+				`    type: "archive"`,
+				`    url: "`+httpServer.URL+`/archive.tar"`,
+				`    refreshPeriod: "1m"`,
+			),
+		},
+	}, func(fileSystem vfs.FS) {
+		ctx := context.Background()
+		system := NewRealSystem(fileSystem)
+
+		readSourceState := func() {
+			s := NewSourceState(
+				WithBaseSystem(system),
+				WithCacheDir(NewAbsPath("/home/user/.cache/chezmoi")),
+				WithDestDir(NewAbsPath("/home/user")),
+				WithSourceDir(NewAbsPath("/home/user/.local/share/chezmoi")),
+				WithSystem(system),
+			)
+			require.NoError(t, s.Read(ctx, readOptions))
+			assert.Equal(t, map[RelPath]External{
+				".dir": {
+					Type:          "archive",
+					URL:           httpServer.URL + "/archive.tar",
+					RefreshPeriod: 1 * time.Minute,
+				},
+			}, s.externals)
+		}
+
+		readSourceState()
+		assert.Equal(t, 1, httpRequests)
+
+		now = now.Add(10 * time.Second)
+		readSourceState()
+		assert.Equal(t, 1, httpRequests)
+
+		now = now.Add(1 * time.Minute)
+		readSourceState()
+		assert.Equal(t, 2, httpRequests)
+	})
+}
+
 func TestSourceStateTargetRelPaths(t *testing.T) {
 	for _, tc := range []struct {
 		name                   string