Variables: Allow to escape "$" with "$$" (#216)

* Variables: Allow to escape "$" with "$$"

* Variables: Added $var style to documentation

Also added missing env expansion on known path config values

Author: jkellerer

config/config_mixins.go

@@ -57,8 +57,11 @@ func (m *mixin) translate(source, variables map[string]interface{}) map[string]i
 
 func (m *mixin) expandVariables(value string, variables map[string]interface{}) string {
 	return os.Expand(value, func(name string) string {
-		lookup := strings.ToUpper(name)
+		if name == "$" {
+			return "$" // allow to escape "$" as "$$"
+		}
 
+		lookup := strings.ToUpper(name)
 		replacement := variables[lookup]
 		if replacement == nil {
 			replacement = m.DefaultVariables[lookup]

config/config_mixins_test.go

@@ -39,8 +39,8 @@ func TestMixin(t *testing.T) {
 			"int", 321,
 			"bool", true,
 			"list", list(1, "2", 3),
-			"list_with_vars", list("${var-1}", "$VAR_2", "[$MyVar]", "-${myvar}-", "-${MYVAR}-"),
-			"string_with_vars", "${var-1} $Var_2 [$MyVar] -${myvar}- -${MYVAR}-",
+			"list_with_vars", list("${var-1}", "$VAR_2", "[$MyVar]", "-${myvar}-", "-${MYVAR}-", "-$${escaped}-"),
+			"string_with_vars", "${var-1} $Var_2 [$MyVar] -${myvar}- -${MYVAR}- $$escaped",
 			"nested_with_vars", list(
 				mm(
 					"list_with_vars", list("${var-1}"),
@@ -58,8 +58,8 @@ func TestMixin(t *testing.T) {
 			"int", 321,
 			"bool", true,
 			"list", list(1, "2", 3),
-			"list_with_vars", list("${var-1}", "${VAR_2}", "[MyDefault]", "-MyDefault-", "-MyDefault-"),
-			"string_with_vars", "${var-1} ${Var_2} [MyDefault] -MyDefault- -MyDefault-",
+			"list_with_vars", list("${var-1}", "${VAR_2}", "[MyDefault]", "-MyDefault-", "-MyDefault-", "-${escaped}-"),
+			"string_with_vars", "${var-1} ${Var_2} [MyDefault] -MyDefault- -MyDefault- $escaped",
 			"nested_with_vars", list(
 				mm(
 					"list_with_vars", list("${var-1}"),
@@ -77,8 +77,8 @@ func TestMixin(t *testing.T) {
 			"int", 321,
 			"bool", true,
 			"list", list(1, "2", 3),
-			"list_with_vars", list("${var-1}", "${VAR_2}", "[MySpecific]", "-MySpecific-", "-MySpecific-"),
-			"string_with_vars", "${var-1} ${Var_2} [MySpecific] -MySpecific- -MySpecific-",
+			"list_with_vars", list("${var-1}", "${VAR_2}", "[MySpecific]", "-MySpecific-", "-MySpecific-", "-${escaped}-"),
+			"string_with_vars", "${var-1} ${Var_2} [MySpecific] -MySpecific- -MySpecific- $escaped",
 			"nested_with_vars", list(
 				mm(
 					"list_with_vars", list("${var-1}"),
@@ -87,7 +87,7 @@ func TestMixin(t *testing.T) {
 				),
 				"${var_2}",
 			),
-		), tpl.Resolve(keysToUpper(mm("myvar", "MySpecific"))))
+		), tpl.Resolve(keysToUpper(mm("myvar", "MySpecific", "escaped", "-"))))
 	})
 
 	t.Run("all-resolved", func(t *testing.T) {
@@ -96,8 +96,8 @@ func TestMixin(t *testing.T) {
 			"int", 321,
 			"bool", true,
 			"list", list(1, "2", 3),
-			"list_with_vars", list("val1", "val2", "[MySpecific]", "-MySpecific-", "-MySpecific-"),
-			"string_with_vars", "val1 val2 [MySpecific] -MySpecific- -MySpecific-",
+			"list_with_vars", list("val1", "val2", "[MySpecific]", "-MySpecific-", "-MySpecific-", "-${escaped}-"),
+			"string_with_vars", "val1 val2 [MySpecific] -MySpecific- -MySpecific- $escaped",
 			"nested_with_vars", list(
 				mm(
 					"list_with_vars", list("val1"),
@@ -111,6 +111,7 @@ func TestMixin(t *testing.T) {
 				"myvar", "MySpecific",
 				"var-1", "val1",
 				"var_2", "val2",
+				"escaped", "-",
 			)),
 		))
 	})

config/global.go

@@ -48,6 +48,9 @@ func NewGlobal() *Global {
 }
 
 func (p *Global) SetRootPath(rootPath string) {
+	p.ShellBinary = fixPaths(p.ShellBinary, expandEnv)
+	p.ResticBinary = fixPath(p.ResticBinary, expandEnv)
+
 	p.SystemdUnitTemplate = fixPath(p.SystemdUnitTemplate, expandEnv, absolutePrefix(rootPath))
 	p.SystemdTimerTemplate = fixPath(p.SystemdTimerTemplate, expandEnv, absolutePrefix(rootPath))
 

config/path.go

@@ -34,7 +34,12 @@ func fixPaths(sources []string, callbacks ...pathFix) (fixed []string) {
 
 func expandEnv(value string) string {
 	if strings.Contains(value, "$") || strings.Contains(value, "%") {
-		value = os.ExpandEnv(value)
+		value = os.Expand(value, func(name string) string {
+			if name == "$" {
+				return "$" // allow to escape "$" as "$$"
+			}
+			return os.Getenv(name)
+		})
 	}
 	return value
 }

config/path_test.go

@@ -86,3 +86,13 @@ func TestFixWindowsPaths(t *testing.T) {
 		assert.Equalf(t, testPath.expected, fixed, "source was '%s'", testPath.source)
 	}
 }
+
+func TestExpandEnv(t *testing.T) {
+	path := os.Getenv("PATH")
+	assert.Equal(t, path, expandEnv("$PATH"))
+	assert.Equal(t, path, expandEnv("${PATH}"))
+	assert.Equal(t, "%PATH%", expandEnv("%PATH%"))
+	assert.Equal(t, "$PATH", expandEnv("$$PATH"))
+	assert.Equal(t, "", expandEnv("${__UNDEFINED_ENV_VAR__}"))
+	assert.Equal(t, "", expandEnv("$__UNDEFINED_ENV_VAR__"))
+}

config/profile.go

@@ -184,7 +184,7 @@ func (b *BackupSection) resolve(p *Profile) {
 }
 
 func (s *BackupSection) setRootPath(p *Profile, rootPath string) {
-	s.SendMonitoringSections.setRootPath(p, rootPath)
+	s.SectionWithScheduleAndMonitoring.setRootPath(p, rootPath)
 
 	s.ExcludeFile = fixPaths(s.ExcludeFile, expandEnv, expandUserHome, absolutePrefix(rootPath))
 	s.FilesFrom = fixPaths(s.FilesFrom, expandEnv, expandUserHome, absolutePrefix(rootPath))
@@ -229,6 +229,11 @@ type SectionWithScheduleAndMonitoring struct {
 	OtherFlagsSection      `mapstructure:",squash"`
 }
 
+func (s *SectionWithScheduleAndMonitoring) setRootPath(p *Profile, rootPath string) {
+	s.SendMonitoringSections.setRootPath(p, rootPath)
+	s.ScheduleBaseSection.setRootPath(p, rootPath)
+}
+
 func (s *SectionWithScheduleAndMonitoring) IsEmpty() bool { return s == nil }
 
 // ScheduleBaseSection contains the parameters for scheduling a command (backup, check, forget, etc.)
@@ -241,6 +246,10 @@ type ScheduleBaseSection struct {
 	ScheduleLockWait   time.Duration `mapstructure:"schedule-lock-wait" show:"noshow" examples:"150s;15m;30m;45m;1h;2h30m" description:"Set the maximum time to wait for acquiring locks when running on schedule"`
 }
 
+func (s *ScheduleBaseSection) setRootPath(_ *Profile, _ string) {
+	s.ScheduleLog = fixPath(s.ScheduleLog, expandEnv, expandUserHome)
+}
+
 func (s *ScheduleBaseSection) GetSchedule() *ScheduleBaseSection { return s }
 
 // CopySection contains the destination parameters for a copy command
@@ -259,7 +268,7 @@ type CopySection struct {
 func (s *CopySection) IsEmpty() bool { return s == nil }
 
 func (c *CopySection) setRootPath(p *Profile, rootPath string) {
-	c.SendMonitoringSections.setRootPath(p, rootPath)
+	c.SectionWithScheduleAndMonitoring.setRootPath(p, rootPath)
 
 	c.PasswordFile = fixPath(c.PasswordFile, expandEnv, expandUserHome, absolutePrefix(rootPath))
 	c.RepositoryFile = fixPath(c.RepositoryFile, expandEnv, expandUserHome, absolutePrefix(rootPath))

docs/content/configuration/variables/index.md

@@ -471,3 +471,87 @@ default {
 
 {{% /tab %}}
 {{% /tabs %}}
+
+
+## Variable expansion in configuration **values**
+
+Variable expansion as described in the previous section using the `{{ .Var }}` syntax refers to [template variables]({{< ref "/configuration/templates" >}}) that are expanded prior to parsing the configuration file. 
+This means they must be used carefully to create correct config markup, but they are also very flexible.
+
+There is also unix style variable expansion using the `${variable}` or `$variable` syntax on configuration **values** that expand after the config file was parsed. Values that take a file path or path expression and a few others support this expansion. 
+
+If not specified differently, these variables resolve to the corresponding environment variable or to an empty value if no such environment variable exists. Exceptions are [mixins]({{< ref "/configuration/inheritance#mixins" >}}) where `$variable` style is used for parametrisation and the profile [config flag]({{< ref "configuration/reference#section-profile" >}}) `prometheus-push-job`.
+
+### Example
+
+Backup current dir (`$PWD`) but prevent backup of `$HOME` where the repository is located:
+
+{{< tabs groupId="config-with-json" >}}
+{{% tab name="toml" %}}
+
+```toml
+version = "1"
+
+[default]
+  repository = "local:${HOME}/backup"
+  password-file = "${HOME}/backup.key"
+
+  [default.backup]
+    source = "$PWD"
+    exclude = ["$HOME/**", ".*", "~*"]
+
+```
+
+{{% /tab %}}
+{{% tab name="yaml" %}}
+
+```yaml
+version: "1"
+
+default:
+  repository: 'local:${HOME}/backup'
+  password-file: '${HOME}/backup.key'
+
+  backup:
+    source: '$PWD'
+    exclude: ['$HOME/**', '.*', '~*']
+
+```
+
+{{% /tab %}}
+{{% tab name="hcl" %}}
+
+```hcl
+default {
+    repository = "local:${HOME}/backup"
+    password-file = "${HOME}/backup.key"
+
+    backup {
+        source = [ "$PWD" ]
+        exclude = [ "$HOME/**", ".*", "~*" ]
+    }
+}
+```
+
+{{% /tab %}}
+{{% tab name="json" %}}
+
+```json
+{
+  "default": {
+    "repository": "local:${HOME}/backup",
+    "password-file": "${HOME}/backup.key",
+    "backup": {
+      "source": [ "$PWD" ],
+      "exclude": [ "$HOME/**", ".*", "~*" ]
+    }
+  }
+}
+```
+
+{{% /tab %}}
+{{% /tabs %}}
+
+{{% notice style="hint" %}}
+Use `$$` to escape a single `$` in configuration values that support variable expansion. E.g. on Windows you might want to exclude `$RECYCLE.BIN`. Specify it as: `exclude = ["$$RECYCLE.BIN"]`.
+{{% /notice %}}

monitor/hook/sender.go

@@ -212,6 +212,9 @@ func resolve(body string, ctx Context) string {
 		case constants.EnvErrorStderr:
 			return ctx.Error.Stderr
 
+		case "$":
+			return "$" // allow to escape "$" as "$$"
+
 		default:
 			return os.Getenv(s)
 		}

monitor/hook/sender_test.go

@@ -3,6 +3,7 @@ package hook
 import (
 	"bytes"
 	"encoding/pem"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -37,6 +38,10 @@ func TestSend(t *testing.T) {
 			Method: http.MethodPost,
 			Body:   "test body\n",
 		}, 1},
+		{config.SendMonitoringSection{
+			Method: http.MethodPost,
+			Body:   "test $$escaped\n",
+		}, 1},
 		{config.SendMonitoringSection{
 			Method: http.MethodPost,
 			Body:   "$PROFILE_NAME\n$PROFILE_COMMAND",
@@ -47,8 +52,8 @@ func TestSend(t *testing.T) {
 		}, 1},
 	}
 
-	for _, testCase := range testCases {
-		t.Run("", func(t *testing.T) {
+	for i, testCase := range testCases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			calls := 0
 			if testCase.cfg.URL.Value() == "" {
 				handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -67,6 +72,7 @@ func TestSend(t *testing.T) {
 					body = strings.ReplaceAll(body, "$ERROR_EXIT_CODE", "test_exit_code")
 					body = strings.ReplaceAll(body, "$ERROR_STDERR", "test_stderr")
 					body = strings.ReplaceAll(body, "$ERROR", "test_error_message")
+					body = strings.ReplaceAll(body, "$$", "$")
 					assert.Equal(t, body, buffer.String())
 					calls++
 				})

monitor/prom/progress.go

@@ -66,6 +66,8 @@ func (p *Progress) Summary(command string, summary monitor.Summary, stderr strin
 		jobName = os.Expand(jobName, func(name string) string {
 			if strings.EqualFold(name, "command") {
 				return command
+			} else if name == "$" {
+				return "$" // allow to escape "$" as "$$"
 			}
 			return ""
 		})