Skip to content

Commit 8218e70

Browse files
jkellererjkellerer
authored
Feature: Hide confidential values in output (#58)
* Config: Introducing confidential value handling Wraps config values that may potentially contain secrets to be hidden. * Integrated confidential values to env and cli * Added test for fmt.Sprintf() not leaking info
1 parent abf0b00 commit 8218e70

12 files changed

+451
-64
lines changed

‎command_error.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ func (c *commandError) Error() string {
2626

2727
func (c *commandError) Commandline() string {
2828
args := ""
29-
if c.scd.args != nil && len(c.scd.args) > 0 {
30-
args = fmt.Sprintf(" \"%s\"", strings.Join(c.scd.args, "\" \""))
29+
argsList := c.scd.publicArgs
30+
if argsList != nil && len(argsList) > 0 {
31+
args = fmt.Sprintf(" \"%s\"", strings.Join(argsList, "\" \""))
3132
}
3233
return fmt.Sprintf("\"%s\"%s", c.scd.command, args)
3334
}

‎config/confidential.go

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package config
2+
3+
import (
4+
"reflect"
5+
"regexp"
6+
)
7+
8+
const ConfidentialReplacement = "***"
9+
10+
// ConfidentialValue is a string value with a public and a confidential representation
11+
type ConfidentialValue struct {
12+
public, confidential string
13+
}
14+
15+
func (c ConfidentialValue) Value() string {
16+
return c.confidential
17+
}
18+
19+
func (c ConfidentialValue) String() string {
20+
return c.public
21+
}
22+
23+
func (c *ConfidentialValue) IsConfidential() bool {
24+
return c.public != c.confidential
25+
}
26+
27+
// hideValue hides the entire value in the public representation
28+
func (c *ConfidentialValue) hideValue() {
29+
if c.IsConfidential() {
30+
return
31+
}
32+
c.public = ConfidentialReplacement
33+
}
34+
35+
// hideSubmatches hides all occurrences of submatches in the public representation
36+
func (c *ConfidentialValue) hideSubmatches(pattern *regexp.Regexp) {
37+
if c.IsConfidential() {
38+
return
39+
}
40+
41+
if matches := pattern.FindStringSubmatchIndex(c.confidential); matches != nil && len(matches) > 2 {
42+
c.public = c.confidential
43+
44+
for i := len(matches) - 2; i > 1; i -= 2 {
45+
start := matches[i]
46+
end := matches[i+1]
47+
48+
c.public = c.public[0:start] + ConfidentialReplacement + c.public[end:]
49+
}
50+
}
51+
}
52+
53+
func NewConfidentialValue(value string) ConfidentialValue {
54+
return ConfidentialValue{public: value, confidential: value}
55+
}
56+
57+
// confidentialValueDecoder implements parsing config parsing for ConfidentialValue
58+
func confidentialValueDecoder() func(from reflect.Type, to reflect.Type, data interface{}) (interface{}, error) {
59+
confidentialValueType := reflect.TypeOf(ConfidentialValue{})
60+
61+
return func(from reflect.Type, to reflect.Type, data interface{}) (interface{}, error) {
62+
if to != confidentialValueType {
63+
return data, nil
64+
}
65+
66+
// Source type may be boolean, numeric or string
67+
values, isset := stringifyValue(reflect.ValueOf(data))
68+
if len(values) == 0 {
69+
if isset {
70+
values = []string{"1"}
71+
} else {
72+
values = []string{"0"}
73+
}
74+
}
75+
76+
return NewConfidentialValue(values[0]), nil
77+
}
78+
}
79+
80+
// See https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html
81+
var (
82+
urlConfidentialPart = regexp.MustCompile("[:/][^:/@]+?:([^:@]+?)@[^:/@]+?") // user:pass@host
83+
urlEnvKeys = regexp.MustCompile("(?i)^.+(_AUTH|_URL)$")
84+
hiddenEnvKeys = regexp.MustCompile("(?i)^(.+_KEY|.+_TOKEN|.*PASSWORD.*|.*SECRET.*)$")
85+
)
86+
87+
// ProcessConfidentialValues hides confidential parts inside the specified Profile.
88+
func ProcessConfidentialValues(profile *Profile) {
89+
if profile == nil {
90+
return
91+
}
92+
93+
// Handle the repo URL
94+
profile.Repository.hideSubmatches(urlConfidentialPart)
95+
96+
// Handle env variables
97+
for name, value := range profile.Environment {
98+
if hiddenEnvKeys.MatchString(name) {
99+
value.hideValue()
100+
profile.Environment[name] = value
101+
} else if urlEnvKeys.MatchString(name) {
102+
value.hideSubmatches(urlConfidentialPart)
103+
profile.Environment[name] = value
104+
}
105+
}
106+
}
107+
108+
// GetNonConfidentialValues returns a new list with confidential values being replaced with their public representation
109+
func GetNonConfidentialValues(profile *Profile, values []string) []string {
110+
if profile == nil {
111+
return values
112+
}
113+
114+
confidentials := []*ConfidentialValue{&profile.Repository}
115+
for _, value := range profile.Environment {
116+
confidentials = append(confidentials, &value)
117+
}
118+
119+
target := make([]string, len(values))
120+
121+
for i := len(values) - 1; i >= 0; i-- {
122+
target[i] = values[i]
123+
124+
for _, c := range confidentials {
125+
if c.IsConfidential() && target[i] == c.Value() {
126+
target[i] = c.String()
127+
}
128+
}
129+
}
130+
131+
return target
132+
}

‎config/confidential_test.go

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
package config
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"reflect"
7+
"regexp"
8+
"strings"
9+
"testing"
10+
11+
"github.com/stretchr/testify/assert"
12+
)
13+
14+
var defaultUrl = "local:user:pass@host"
15+
var defaultUrlReplaced = fmt.Sprintf("local:user:%s@host", ConfidentialReplacement)
16+
17+
func TestConfidentialHideAll(t *testing.T) {
18+
value := NewConfidentialValue("val")
19+
20+
assert.Equal(t, value.String(), value.Value())
21+
assert.Equal(t, "val", value.String())
22+
23+
value.hideValue()
24+
assert.Equal(t, "val", value.Value())
25+
assert.Equal(t, ConfidentialReplacement, value.String())
26+
}
27+
28+
func TestConfidentialHideSubmatch(t *testing.T) {
29+
value := NewConfidentialValue("some-vAl-with-sEcRet-parts")
30+
31+
assert.Equal(t, value.String(), value.Value())
32+
assert.Equal(t, "some-vAl-with-sEcRet-parts", value.String())
33+
34+
value.hideSubmatches(regexp.MustCompile("(?i).+(val).+(secret).+"))
35+
assert.Equal(t, "some-vAl-with-sEcRet-parts", value.Value())
36+
37+
expected := fmt.Sprintf("some-%s-with-%s-parts", ConfidentialReplacement, ConfidentialReplacement)
38+
assert.Equal(t, expected, value.String())
39+
}
40+
41+
func TestFmtStringDoesntLeakConfidentialValues(t *testing.T) {
42+
value := NewConfidentialValue("secret")
43+
value.hideValue()
44+
45+
assert.Equal(t, ConfidentialReplacement, fmt.Sprintf("%s", value))
46+
assert.Equal(t, ConfidentialReplacement, fmt.Sprintf("%v", value))
47+
assert.Equal(t, ConfidentialReplacement, value.String())
48+
assert.Equal(t, "secret", value.Value())
49+
}
50+
51+
func TestStringifyPassesConfidentialValues(t *testing.T) {
52+
value := NewConfidentialValue("secret")
53+
value.hideValue()
54+
55+
v1, _ := stringifyValue(reflect.ValueOf(value))
56+
v2, _ := stringifyConfidentialValue(reflect.ValueOf(value))
57+
assert.Equal(t, []string{ConfidentialReplacement}, v1)
58+
assert.Equal(t, []string{"secret"}, v2)
59+
}
60+
61+
func TestConfidentialURLs(t *testing.T) {
62+
// https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html
63+
urls := map[string]string{
64+
"local:some/path": "-",
65+
"sftp:user@host:/srv/restic-repo": "-",
66+
"sftp://user@[::1]:2222//srv/restic-repo": "-",
67+
"sftp:restic-backup-host:/srv/restic-repo": "-",
68+
"rest:http://host:8000/": "-",
69+
"rest:https://user:1234fdfASDasfwY.-+;@host:8000/": fmt.Sprintf("rest:https://user:%s@host:8000/", ConfidentialReplacement),
70+
"rest:https://user:35%3Asad%C3%B6p%C3%9F@host:8000/": fmt.Sprintf("rest:https://user:%s@host:8000/", ConfidentialReplacement),
71+
"rest:https://user:pass@host:8000/f/": fmt.Sprintf("rest:https://user:%s@host:8000/f/", ConfidentialReplacement),
72+
"s3:s3.amazonaws.com/bucket_name": "-",
73+
"s3:https://<WASABI-SERVICE-URL>/<WASABI-BUCKET-NAME>": "-",
74+
"swift:container_name:/path": "-",
75+
"azure:foo:/": "-",
76+
"gs:foo:/": "-",
77+
"rclone:b2prod:yggdrasil/foo/bar/baz": "-",
78+
}
79+
80+
for url, expected := range urls {
81+
testConfig := fmt.Sprintf(`
82+
[profile]
83+
repository = "%s"
84+
`, url)
85+
86+
profile, err := getProfile("toml", testConfig, "profile")
87+
assert.Nil(t, err)
88+
assert.NotNil(t, profile)
89+
90+
if expected == "-" {
91+
expected = url
92+
}
93+
assert.Equal(t, expected, profile.Repository.String())
94+
assert.Equal(t, url, profile.Repository.Value())
95+
}
96+
}
97+
98+
func TestConfidentialEnvironment(t *testing.T) {
99+
// https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html
100+
vars := map[string]string{
101+
"MY_VALUE": "-",
102+
"MY_KEY": "*",
103+
"PASSWORD": "*",
104+
"MY_URL": "<url>",
105+
// AWS, MinIO, Wasabi, Alibaba Cloud
106+
"AWS_ACCESS_KEY_ID": "-",
107+
"AWS_SECRET_ACCESS_KEY": "*",
108+
"AWS_DEFAULT_REGION": "-",
109+
// OpenStack Swift
110+
"ST_AUTH": "<url>",
111+
"ST_USER": "-",
112+
"ST_KEY": "*",
113+
"OS_AUTH_URL": "<url>",
114+
"OS_REGION_NAME": "-",
115+
"OS_USERNAME": "-",
116+
"OS_PASSWORD": "*",
117+
"OS_TENANT_ID": "-",
118+
"OS_TENANT_NAME": "-",
119+
"OS_USER_ID": "-",
120+
"OS_USER_DOMAIN_NAME": "-",
121+
"OS_USER_DOMAIN_ID": "-",
122+
"OS_PROJECT_NAME": "-",
123+
"OS_PROJECT_DOMAIN_NAME": "-",
124+
"OS_PROJECT_DOMAIN_ID": "-",
125+
"OS_TRUST_ID": "-",
126+
"OS_APPLICATION_CREDENTIAL_ID": "-",
127+
"OS_APPLICATION_CREDENTIAL_NAME": "-",
128+
"OS_APPLICATION_CREDENTIAL_SECRET": "*",
129+
"OS_STORAGE_URL": "<url>",
130+
"OS_AUTH_TOKEN": "*",
131+
"SWIFT_DEFAULT_CONTAINER_POLICY": "-",
132+
// Backblaze B2
133+
"B2_ACCOUNT_ID": "-",
134+
"B2_ACCOUNT_KEY": "*",
135+
// Microsoft Azure Blob Storage
136+
"AZURE_ACCOUNT_NAME": "-",
137+
"AZURE_ACCOUNT_KEY": "*",
138+
// Google Cloud Storage
139+
"GOOGLE_PROJECT_ID": "-",
140+
"GOOGLE_APPLICATION_CREDENTIALS": "-",
141+
"GOOGLE_ACCESS_TOKEN": "*",
142+
}
143+
144+
for name, expected := range vars {
145+
testConfig := fmt.Sprintf(`
146+
[profile.env]
147+
%s = "%s"
148+
`, name, defaultUrl)
149+
150+
profile, err := getProfile("toml", testConfig, "profile")
151+
assert.Nil(t, err)
152+
assert.NotNil(t, profile)
153+
154+
switch expected {
155+
case "<url>":
156+
expected = defaultUrlReplaced
157+
case "*":
158+
expected = ConfidentialReplacement
159+
default:
160+
expected = defaultUrl
161+
}
162+
163+
name = strings.ToLower(name)
164+
env := profile.Environment[name]
165+
assert.Equal(t, expected, env.String())
166+
assert.Equal(t, defaultUrl, env.Value())
167+
}
168+
}
169+
170+
func TestShowConfigHidesConfidentialValues(t *testing.T) {
171+
testConfig := `
172+
profile:
173+
repository: "local:user:pass@host"
174+
env:
175+
MY_VALUE: "val"
176+
MY_URL: "local:user:pass@host"
177+
MY_KEY: 1234
178+
MY_TOKEN: false
179+
MY_PASSWORD: "otherval"
180+
`
181+
profile, err := getProfile("yaml", testConfig, "profile")
182+
assert.Nil(t, err)
183+
assert.NotNil(t, profile)
184+
185+
buffer := &bytes.Buffer{}
186+
assert.Nil(t, ShowStruct(buffer, profile, "p"))
187+
188+
result := regexp.MustCompile("\\s+").ReplaceAllString(buffer.String(), " ")
189+
result = strings.TrimSpace(result)
190+
191+
assert.Contains(t, result, "my_value: val")
192+
assert.Contains(t, result, "my_url: "+defaultUrlReplaced)
193+
assert.Contains(t, result, "my_key: "+ConfidentialReplacement)
194+
assert.Contains(t, result, "my_token: "+ConfidentialReplacement)
195+
assert.Contains(t, result, "my_password: "+ConfidentialReplacement)
196+
assert.Contains(t, result, "repository: "+defaultUrlReplaced)
197+
}
198+
199+
func TestGetNonConfidentialValues(t *testing.T) {
200+
testConfig := `
201+
profile:
202+
verbose: false
203+
repository: "local:user:pass@host"
204+
env:
205+
MY_PASSWORD: "otherval"
206+
`
207+
profile, err := getProfile("yaml", testConfig, "profile")
208+
assert.Nil(t, err)
209+
assert.NotNil(t, profile)
210+
211+
result := GetNonConfidentialValues(profile, []string{"a", defaultUrl, "b", "otherval", "c"})
212+
assert.Equal(t, []string{"a", defaultUrlReplaced, "b", ConfidentialReplacement, "c"}, result)
213+
}

‎config/config.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,12 @@ type Config struct {
4545
var (
4646
configOption = viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
4747
mapstructure.StringToTimeDurationHookFunc(),
48+
confidentialValueDecoder(),
4849
))
4950

5051
configOptionHCL = viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
5152
mapstructure.StringToTimeDurationHookFunc(),
53+
confidentialValueDecoder(),
5254
sliceOfMapsToMapHookFunc(),
5355
))
5456
)
@@ -305,6 +307,7 @@ func (c *Config) getProfile(profileKey string) (*Profile, error) {
305307
if err != nil {
306308
return nil, err
307309
}
310+
308311
if profile.Inherit != "" {
309312
inherit := profile.Inherit
310313
// Load inherited profile
@@ -323,6 +326,10 @@ func (c *Config) getProfile(profileKey string) (*Profile, error) {
323326
// make sure it has the right name
324327
profile.Name = profileKey
325328
}
329+
330+
// Hide confidential values (keys, passwords) from the public representation
331+
ProcessConfidentialValues(profile)
332+
326333
return profile, nil
327334
}
328335

0 commit comments

Comments
 (0)