From a0285982fe41e29aa086ae5ba067c8ebf7c564c2 Mon Sep 17 00:00:00 2001 From: Tom Payne Date: Sat, 12 Aug 2023 22:01:41 +0200 Subject: [PATCH] feat: Restore --autotemplate flag to add command This reverts commit ed0c798a817eefe86b4522de305d9cc72f5de401. --- .../chezmoi.io/docs/reference/commands/add.md | 13 ++ internal/chezmoi/autotemplate.go | 139 ++++++++++++ internal/chezmoi/autotemplate_test.go | 214 ++++++++++++++++++ internal/chezmoi/sourcestate.go | 10 + internal/chezmoi/sourcestate_test.go | 17 ++ internal/cmd/addcmd.go | 9 + .../testdata/scripts/addautotemplate.txtar | 34 +++ 7 files changed, 436 insertions(+) create mode 100644 internal/chezmoi/autotemplate.go create mode 100644 internal/chezmoi/autotemplate_test.go create mode 100644 internal/cmd/testdata/scripts/addautotemplate.txtar diff --git a/assets/chezmoi.io/docs/reference/commands/add.md b/assets/chezmoi.io/docs/reference/commands/add.md index 5d55d4005c0..11031e956e1 100644 --- a/assets/chezmoi.io/docs/reference/commands/add.md +++ b/assets/chezmoi.io/docs/reference/commands/add.md @@ -4,6 +4,19 @@ Add *target*s to the source state. If any target is already in the source state, then its source state is replaced with its current state in the destination directory. +## `--autotemplate` (deprecated) + +Automatically generate a template by replacing strings that match variable +values from the `data` section of the config file with their respective config +names as a template string. Longer substitutions occur before shorter ones. +This implies the `--template` option. + +!!! warning + + `--autotemplate` uses a greedy algorithm which occasionally generates + templates with unwanted variable substitutions. Carefully review any + templates it generates. + ## `--encrypt` Encrypt files using the defined encryption method. diff --git a/internal/chezmoi/autotemplate.go b/internal/chezmoi/autotemplate.go new file mode 100644 index 00000000000..4e05dec03c2 --- /dev/null +++ b/internal/chezmoi/autotemplate.go @@ -0,0 +1,139 @@ +package chezmoi + +import ( + "regexp" + "sort" + "strings" + + "golang.org/x/exp/slices" +) + +// A templateVariable is a template variable. It is used instead of a +// map[string]string so that we can control order. +type templateVariable struct { + components []string + value string +} + +var templateMarkerRx = regexp.MustCompile(`\{{2,}|\}{2,}`) + +// autoTemplate converts contents into a template by escaping template markers +// and replacing values in data with their keys. It returns the template and if +// any replacements were made. +func autoTemplate(contents []byte, data map[string]any) ([]byte, bool) { + contentsStr := string(contents) + replacements := false + + // Replace template markers. + replacedTemplateMarkersStr := templateMarkerRx.ReplaceAllString(contentsStr, `{{ "$0" }}`) + if replacedTemplateMarkersStr != contentsStr { + contentsStr = replacedTemplateMarkersStr + replacements = true + } + + // Determine the priority order of replacements. + // + // Replace longest values first. If there are multiple matches for the same + // length of value, then choose the shallowest first so that .variable is + // preferred over .chezmoi.config.data.variable. If there are multiple + // matches at the same depth, chose the variable that comes first + // alphabetically. + variables := extractVariables(data) + sort.Slice(variables, func(i, j int) bool { + // First sort by value length, longest first. + valueI := variables[i].value + valueJ := variables[j].value + switch { + case len(valueI) > len(valueJ): + return true + case len(valueI) == len(valueJ): + // Second sort by value name depth, shallowest first. + componentsI := variables[i].components + componentsJ := variables[j].components + switch { + case len(componentsI) < len(componentsJ): + return true + case len(componentsI) == len(componentsJ): + // Thirdly, sort by component names in alphabetical order. + return slices.Compare(componentsI, componentsJ) < 0 + default: + return false + } + default: + return false + } + }) + + // Replace variables in order. + // + // This naive approach will generate incorrect templates if the variable + // names match variable values. The algorithm here is probably O(N^2), we + // can do better. + for _, variable := range variables { + if variable.value == "" { + continue + } + + index := strings.Index(contentsStr, variable.value) + for index != -1 && index != len(contentsStr) { + if !inWord(contentsStr, index) && !inWord(contentsStr, index+len(variable.value)) { + // Replace variable.value which is on word boundaries at both + // ends. + replacement := "{{ ." + strings.Join(variable.components, ".") + " }}" + contentsStr = contentsStr[:index] + replacement + contentsStr[index+len(variable.value):] + index += len(replacement) + replacements = true + } else { + // Otherwise, keep looking. Consume at least one byte so we make + // progress. + index++ + } + + // Look for the next occurrence of variable.value. + j := strings.Index(contentsStr[index:], variable.value) + if j == -1 { + // No more occurrences found, so terminate the loop. + break + } + // Advance to the next occurrence. + index += j + } + } + + return []byte(contentsStr), replacements +} + +// appendVariables appends all template variables in data to variables +// and returns variables. data is assumed to be rooted at parent. +func appendVariables( + variables []templateVariable, parent []string, data map[string]any, +) []templateVariable { + for name, value := range data { + switch value := value.(type) { + case string: + variable := templateVariable{ + components: append(slices.Clone(parent), name), + value: value, + } + variables = append(variables, variable) + case map[string]any: + variables = appendVariables(variables, append(parent, name), value) + } + } + return variables +} + +// extractVariables extracts all template variables from data. +func extractVariables(data map[string]any) []templateVariable { + return appendVariables(nil, nil, data) +} + +// inWord returns true if splitting s at position i would split a word. +func inWord(s string, i int) bool { + return i > 0 && i < len(s) && isWord(s[i-1]) && isWord(s[i]) +} + +// isWord returns true if b is a word byte. +func isWord(b byte) bool { + return '0' <= b && b <= '9' || 'A' <= b && b <= 'Z' || 'a' <= b && b <= 'z' +} diff --git a/internal/chezmoi/autotemplate_test.go b/internal/chezmoi/autotemplate_test.go new file mode 100644 index 00000000000..b09cdfc48ae --- /dev/null +++ b/internal/chezmoi/autotemplate_test.go @@ -0,0 +1,214 @@ +package chezmoi + +import ( + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestAutoTemplate(t *testing.T) { + for _, tc := range []struct { + name string + contentsStr string + data map[string]any + expected string + expectedReplacements bool + }{ + { + name: "simple", + contentsStr: "email = you@example.com\n", + data: map[string]any{ + "email": "you@example.com", + }, + expected: "email = {{ .email }}\n", + expectedReplacements: true, + }, + { + name: "longest_first", + contentsStr: "name = John Smith\nfirstName = John\n", + data: map[string]any{ + "name": "John Smith", + "firstName": "John", + }, + expected: "" + + "name = {{ .name }}\n" + + "firstName = {{ .firstName }}\n", + expectedReplacements: true, + }, + { + name: "alphabetical_first", + contentsStr: "name = John Smith\n", + data: map[string]any{ + "alpha": "John Smith", + "beta": "John Smith", + "gamma": "John Smith", + }, + expected: "name = {{ .alpha }}\n", + expectedReplacements: true, + }, + { + name: "nested_values", + contentsStr: "email = you@example.com\n", + data: map[string]any{ + "personal": map[string]any{ + "email": "you@example.com", + }, + }, + expected: "email = {{ .personal.email }}\n", + expectedReplacements: true, + }, + { + name: "only_replace_words", + contentsStr: "darwinian evolution", + data: map[string]any{ + "os": "darwin", + }, + expected: "darwinian evolution", // not "{{ .os }}ian evolution" + }, + { + name: "longest_match_first", + contentsStr: "/home/user", + data: map[string]any{ + "homeDir": "/home/user", + }, + expected: "{{ .homeDir }}", + expectedReplacements: true, + }, + { + name: "longest_match_first_prefix", + contentsStr: "HOME=/home/user", + data: map[string]any{ + "homeDir": "/home/user", + }, + expected: "HOME={{ .homeDir }}", + expectedReplacements: true, + }, + { + name: "longest_match_first_suffix", + contentsStr: "/home/user/something", + data: map[string]any{ + "homeDir": "/home/user", + }, + expected: "{{ .homeDir }}/something", + expectedReplacements: true, + }, + { + name: "longest_match_first_prefix_and_suffix", + contentsStr: "HOME=/home/user/something", + data: map[string]any{ + "homeDir": "/home/user", + }, + expected: "HOME={{ .homeDir }}/something", + expectedReplacements: true, + }, + { + name: "depth_first", + contentsStr: "a", + data: map[string]any{ + "deep": map[string]any{ + "deeper": "a", + }, + "shallow": "a", + }, + expected: "{{ .shallow }}", + expectedReplacements: true, + }, + { + name: "alphabetical_first", + contentsStr: "a", + data: map[string]any{ + "parent": map[string]any{ + "alpha": "a", + "beta": "a", + }, + }, + expected: "{{ .parent.alpha }}", + expectedReplacements: true, + }, + { + name: "words_only", + contentsStr: "aaa aa a aa aaa aa a aa aaa", + data: map[string]any{ + "alpha": "a", + }, + expected: "aaa aa {{ .alpha }} aa aaa aa {{ .alpha }} aa aaa", + expectedReplacements: true, + }, + { + name: "words_only_2", + contentsStr: "aaa aa a aa aaa aa a aa aaa", + data: map[string]any{ + "alpha": "aa", + }, + expected: "aaa {{ .alpha }} a {{ .alpha }} aaa {{ .alpha }} a {{ .alpha }} aaa", + expectedReplacements: true, + }, + { + name: "words_only_3", + contentsStr: "aaa aa a aa aaa aa a aa aaa", + data: map[string]any{ + "alpha": "aaa", + }, + expected: "{{ .alpha }} aa a aa {{ .alpha }} aa a aa {{ .alpha }}", + expectedReplacements: true, + }, + { + name: "skip_empty", + contentsStr: "a", + data: map[string]any{ + "empty": "", + }, + expected: "a", + }, + { + name: "markers", + contentsStr: "{{}}", + expected: `{{ "{{" }}{{ "}}" }}`, + expectedReplacements: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + actualTemplate, actualReplacements := autoTemplate([]byte(tc.contentsStr), tc.data) + assert.Equal(t, tc.expected, string(actualTemplate)) + assert.Equal(t, tc.expectedReplacements, actualReplacements) + }) + } +} + +func TestInWord(t *testing.T) { + for _, tc := range []struct { + s string + i int + expected bool + }{ + {s: "", i: 0, expected: false}, + {s: "a", i: 0, expected: false}, + {s: "a", i: 1, expected: false}, + {s: "ab", i: 0, expected: false}, + {s: "ab", i: 1, expected: true}, + {s: "ab", i: 2, expected: false}, + {s: "abc", i: 0, expected: false}, + {s: "abc", i: 1, expected: true}, + {s: "abc", i: 2, expected: true}, + {s: "abc", i: 3, expected: false}, + {s: " abc ", i: 0, expected: false}, + {s: " abc ", i: 1, expected: false}, + {s: " abc ", i: 2, expected: true}, + {s: " abc ", i: 3, expected: true}, + {s: " abc ", i: 4, expected: false}, + {s: " abc ", i: 5, expected: false}, + {s: "/home/user", i: 0, expected: false}, + {s: "/home/user", i: 1, expected: false}, + {s: "/home/user", i: 2, expected: true}, + {s: "/home/user", i: 3, expected: true}, + {s: "/home/user", i: 4, expected: true}, + {s: "/home/user", i: 5, expected: false}, + {s: "/home/user", i: 6, expected: false}, + {s: "/home/user", i: 7, expected: true}, + {s: "/home/user", i: 8, expected: true}, + {s: "/home/user", i: 9, expected: true}, + {s: "/home/user", i: 10, expected: false}, + } { + assert.Equal(t, tc.expected, inWord(tc.s, tc.i)) + } +} diff --git a/internal/chezmoi/sourcestate.go b/internal/chezmoi/sourcestate.go index 53d31a6df38..4797078a50c 100644 --- a/internal/chezmoi/sourcestate.go +++ b/internal/chezmoi/sourcestate.go @@ -298,6 +298,7 @@ type ReplaceFunc func(targetRelPath RelPath, newSourceStateEntry, oldSourceState // AddOptions are options to SourceState.Add. type AddOptions struct { + AutoTemplate bool // Automatically create templates, if possible. Create bool // Add create_ entries instead of normal entries. Encrypt bool // Encrypt files. EncryptedSuffix string // Suffix for encrypted files. @@ -2109,6 +2110,13 @@ func (s *SourceState) newSourceStateFileEntryFromFile( if err != nil { return nil, err } + if options.AutoTemplate { + var replacements bool + contents, replacements = autoTemplate(contents, s.TemplateData()) + if replacements { + fileAttr.Template = true + } + } if len(contents) == 0 { fileAttr.Empty = true } @@ -2148,6 +2156,8 @@ func (s *SourceState) newSourceStateFileEntryFromSymlink( contents := []byte(linkname) template := false switch { + case options.AutoTemplate: + contents, template = autoTemplate(contents, s.TemplateData()) case options.Template: template = true case !options.Template && options.TemplateSymlinks: diff --git a/internal/chezmoi/sourcestate_test.go b/internal/chezmoi/sourcestate_test.go index 851ff8a403b..1ad396c994f 100644 --- a/internal/chezmoi/sourcestate_test.go +++ b/internal/chezmoi/sourcestate_test.go @@ -427,6 +427,23 @@ func TestSourceStateAdd(t *testing.T) { ), }, }, + { + name: "template", + destAbsPaths: []AbsPath{ + NewAbsPath("/home/user/.template"), + }, + addOptions: AddOptions{ + AutoTemplate: true, + Filter: NewEntryTypeFilter(EntryTypesAll, EntryTypesNone), + }, + tests: []any{ + vfst.TestPath("/home/user/.local/share/chezmoi/dot_template.tmpl", + vfst.TestModeIsRegular, + vfst.TestModePerm(0o666&^chezmoitest.Umask), + vfst.TestContentsString("key = {{ .variable }}\n"), + ), + }, + }, { name: "dir_and_dir_file", destAbsPaths: []AbsPath{ diff --git a/internal/cmd/addcmd.go b/internal/cmd/addcmd.go index fc6e0a32688..07b63f5afbc 100644 --- a/internal/cmd/addcmd.go +++ b/internal/cmd/addcmd.go @@ -10,6 +10,7 @@ import ( type addCmdConfig struct { TemplateSymlinks bool `json:"templateSymlinks" mapstructure:"templateSymlinks" yaml:"templateSymlinks"` + autoTemplate bool create bool encrypt bool exact bool @@ -39,6 +40,13 @@ func (c *Config) newAddCmd() *cobra.Command { } flags := addCmd.Flags() + flags.BoolVarP( + &c.Add.autoTemplate, + "autotemplate", + "a", + c.Add.autoTemplate, + "Generate the template when adding files as templates", + ) //nolint:lll flags.BoolVar( &c.Add.create, "create", @@ -188,6 +196,7 @@ func (c *Config) runAddCmd( c.destSystem, destAbsPathInfos, &chezmoi.AddOptions{ + AutoTemplate: c.Add.autoTemplate, Create: c.Add.create, Encrypt: c.Add.encrypt, EncryptedSuffix: c.encryption.EncryptedSuffix(), diff --git a/internal/cmd/testdata/scripts/addautotemplate.txtar b/internal/cmd/testdata/scripts/addautotemplate.txtar new file mode 100644 index 00000000000..e7b477c42e8 --- /dev/null +++ b/internal/cmd/testdata/scripts/addautotemplate.txtar @@ -0,0 +1,34 @@ +# test that chezmoi add --autotemplate on a file with a replacement creates a template in the source directory +exec chezmoi add --autotemplate $HOME${/}.template +cmp $CHEZMOISOURCEDIR/dot_template.tmpl golden/dot_template.tmpl + +# test that chezmoi add --autotemplate on a symlink with a replacement creates a template in the source directory +symlink $HOME/.symlink -> .target-value +exec chezmoi add --autotemplate $HOME${/}.symlink +cmp $CHEZMOISOURCEDIR/symlink_dot_symlink.tmpl golden/symlink_dot_symlink.tmpl + +# test that chezmoi add --autotemplate does not create a template if no replacements occurred +exec chezmoi add --autotemplate $HOME${/}.file +cmp $CHEZMOISOURCEDIR/dot_file golden/dot_file + +# test that chezmoi add --autotemplate escapes brackets +exec chezmoi add --autotemplate $HOME${/}.vimrc +cmp $CHEZMOISOURCEDIR/dot_vimrc.tmpl golden/dot_vimrc.tmpl + +-- golden/dot_file -- +# contents of .file +-- golden/dot_template.tmpl -- +key = {{ .variable }} +-- golden/dot_vimrc.tmpl -- +set foldmarker={{ "{{" }},{{ "}}" }} +-- golden/symlink_dot_symlink.tmpl -- +.target-{{ .variable }} +-- home/user/.config/chezmoi/chezmoi.toml -- +[data] + variable = "value" +-- home/user/.file -- +# contents of .file +-- home/user/.template -- +key = value +-- home/user/.vimrc -- +set foldmarker={{,}}