From f418468c87d3071c5d9ed14ce850996c77251080 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 22 Feb 2017 18:54:49 -0500 Subject: [PATCH 1/4] Add an AEAD encrypting transformer for storing secrets encrypted at rest Tweak the ValueTransformer interface slightly to support additional context information (to allow authenticated data to be generated by the store and passed to the transformer). Add a prefix transformer that looks for known matching prefixes and uses them. Add an AES GCM transformer that performs AEAD on the values coming in and out of the store. --- .../pkg/storage/value/encrypt/aes/aes.go | 80 ++++++++ .../pkg/storage/value/encrypt/aes/aes_test.go | 172 ++++++++++++++++++ .../pkg/storage/value/transformer.go | 146 +++++++++++++++ .../pkg/storage/value/transformer_test.go | 101 ++++++++++ 4 files changed, 499 insertions(+) create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes_test.go create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go new file mode 100644 index 000000000000..8a180eb3c4ba --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go @@ -0,0 +1,80 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package aes transforms values for storage at rest using AES-GCM. +package aes + +import ( + "crypto/cipher" + "crypto/rand" + "fmt" + + "k8s.io/apiserver/pkg/storage/value" +) + +// gcm implements AEAD encryption of the provided values given a cipher.Block algorithm. +// The authenticated data provided as part of the value.Context method must match when the same +// value is set to and loaded from storage. In order to ensure that values cannot be copied by +// an attacker from a location under their control, use characteristics of the storage location +// (such as the etcd key) as part of the authenticated data. +// +// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys +// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D +// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same +// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and +// therefore transformers using this implementation *must* ensure they allow for frequent key +// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to +// random nonces. +type gcm struct { + block cipher.Block +} + +// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given +// data. +func NewGCMTransformer(block cipher.Block) value.Transformer { + return &gcm{block: block} +} + +func (t *gcm) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) { + aead, err := cipher.NewGCM(t.block) + if err != nil { + return nil, false, err + } + nonceSize := aead.NonceSize() + if len(data) < nonceSize { + return nil, false, fmt.Errorf("the stored data was shorter than the required size") + } + result, err := aead.Open(nil, data[:nonceSize], data[nonceSize:], context.AuthenticatedData()) + return result, false, err +} + +func (t *gcm) TransformToStorage(data []byte, context value.Context) ([]byte, error) { + aead, err := cipher.NewGCM(t.block) + if err != nil { + return nil, err + } + nonceSize := aead.NonceSize() + result := make([]byte, nonceSize+aead.Overhead()+len(data)) + n, err := rand.Read(result[:nonceSize]) + if err != nil { + return nil, err + } + if n != nonceSize { + return nil, fmt.Errorf("unable to read sufficient random bytes") + } + cipherText := aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], data, context.AuthenticatedData()) + return result[:nonceSize+len(cipherText)], nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes_test.go new file mode 100644 index 000000000000..ea2ea3936f1d --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aes + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "fmt" + "testing" + + "k8s.io/apiserver/pkg/storage/value" +) + +func TestGCMDataStable(t *testing.T) { + block, err := aes.NewCipher([]byte("0123456789abcdef")) + if err != nil { + t.Fatal(err) + } + aead, err := cipher.NewGCM(block) + if err != nil { + t.Fatal(err) + } + // IMPORTANT: If you must fix this test, then all previously encrypted data from previously compiled versions is broken unless you hardcode the nonce size to 12 + if aead.NonceSize() != 12 { + t.Fatalf("The underlying Golang crypto size has changed, old version of AES on disk will not be readable unless the AES implementation is changed to hardcode nonce size.") + } +} + +func TestGCMKeyRotation(t *testing.T) { + testErr := fmt.Errorf("test error") + block1, err := aes.NewCipher([]byte("abcdefghijklmnop")) + if err != nil { + t.Fatal(err) + } + block2, err := aes.NewCipher([]byte("0123456789abcdef")) + if err != nil { + t.Fatal(err) + } + + context := value.DefaultContext([]byte("authenticated_data")) + + p := value.NewPrefixTransformers(testErr, + value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)}, + value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)}, + ) + out, err := p.TransformToStorage([]byte("firstvalue"), context) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("first:")) { + t.Fatalf("unexpected prefix: %q", out) + } + from, stale, err := p.TransformFromStorage(out, context) + if err != nil { + t.Fatal(err) + } + if stale || !bytes.Equal([]byte("firstvalue"), from) { + t.Fatalf("unexpected data: %t %q", stale, from) + } + + // verify changing the context fails storage + from, stale, err = p.TransformFromStorage(out, value.DefaultContext([]byte("incorrect_context"))) + if err == nil { + t.Fatalf("expected unauthenticated data") + } + + // reverse the order, use the second key + p = value.NewPrefixTransformers(testErr, + value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)}, + value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)}, + ) + from, stale, err = p.TransformFromStorage(out, context) + if err != nil { + t.Fatal(err) + } + if !stale || !bytes.Equal([]byte("firstvalue"), from) { + t.Fatalf("unexpected data: %t %q", stale, from) + } +} + +func BenchmarkGCMRead_16_1024(b *testing.B) { benchmarkGCMRead(b, 16, 1024, false) } +func BenchmarkGCMRead_32_1024(b *testing.B) { benchmarkGCMRead(b, 32, 1024, false) } +func BenchmarkGCMRead_32_16384(b *testing.B) { benchmarkGCMRead(b, 32, 16384, false) } +func BenchmarkGCMRead_32_16384_Stale(b *testing.B) { benchmarkGCMRead(b, 32, 16384, true) } + +func BenchmarkGCMWrite_16_1024(b *testing.B) { benchmarkGCMWrite(b, 16, 1024) } +func BenchmarkGCMWrite_32_1024(b *testing.B) { benchmarkGCMWrite(b, 32, 1024) } +func BenchmarkGCMWrite_32_16384(b *testing.B) { benchmarkGCMWrite(b, 32, 16384) } + +func benchmarkGCMRead(b *testing.B, keyLength int, valueLength int, stale bool) { + block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength)) + if err != nil { + b.Fatal(err) + } + block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength)) + if err != nil { + b.Fatal(err) + } + p := value.NewPrefixTransformers(nil, + value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)}, + value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)}, + ) + + context := value.DefaultContext([]byte("authenticated_data")) + v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16) + + out, err := p.TransformToStorage(v, context) + if err != nil { + b.Fatal(err) + } + // reverse the key order if stale + if stale { + p = value.NewPrefixTransformers(nil, + value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)}, + value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)}, + ) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + from, stale, err := p.TransformFromStorage(out, context) + if err != nil { + b.Fatal(err) + } + if stale { + b.Fatalf("unexpected data: %t %q", stale, from) + } + } + b.StopTimer() +} + +func benchmarkGCMWrite(b *testing.B, keyLength int, valueLength int) { + block1, err := aes.NewCipher(bytes.Repeat([]byte("a"), keyLength)) + if err != nil { + b.Fatal(err) + } + block2, err := aes.NewCipher(bytes.Repeat([]byte("b"), keyLength)) + if err != nil { + b.Fatal(err) + } + p := value.NewPrefixTransformers(nil, + value.PrefixTransformer{Prefix: []byte("first:"), Transformer: NewGCMTransformer(block1)}, + value.PrefixTransformer{Prefix: []byte("second:"), Transformer: NewGCMTransformer(block2)}, + ) + + context := value.DefaultContext([]byte("authenticated_data")) + v := bytes.Repeat([]byte("0123456789abcdef"), valueLength/16) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := p.TransformToStorage(v, context) + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go new file mode 100644 index 000000000000..ab5d4af959bf --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer.go @@ -0,0 +1,146 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package value contains methods for assisting with transformation of values in storage. +package value + +import ( + "bytes" + "fmt" + "sync" +) + +// Context is additional information that a storage transformation may need to verify the data at rest. +type Context interface { + // AuthenticatedData should return an array of bytes that describes the current value. If the value changes, + // the transformer may report the value as unreadable or tampered. This may be nil if no such description exists + // or is needed. For additional verification, set this to data that strongly identifies the value, such as + // the key and creation version of the stored data. + AuthenticatedData() []byte +} + +// Transformer allows a value to be transformed before being read from or written to the underlying store. The methods +// must be able to undo the transformation caused by the other. +type Transformer interface { + // TransformFromStorage may transform the provided data from its underlying storage representation or return an error. + // Stale is true if the object on disk is stale and a write to etcd should be issued, even if the contents of the object + // have not changed. + TransformFromStorage(data []byte, context Context) (out []byte, stale bool, err error) + // TransformToStorage may transform the provided data into the appropriate form in storage or return an error. + TransformToStorage(data []byte, context Context) (out []byte, err error) +} + +type identityTransformer struct{} + +// IdentityTransformer performs no transformation of the provided data. +var IdentityTransformer Transformer = identityTransformer{} + +func (identityTransformer) TransformFromStorage(b []byte, ctx Context) ([]byte, bool, error) { + return b, false, nil +} +func (identityTransformer) TransformToStorage(b []byte, ctx Context) ([]byte, error) { + return b, nil +} + +// DefaultContext is a simple implementation of Context for a slice of bytes. +type DefaultContext []byte + +// AuthenticatedData returns itself. +func (c DefaultContext) AuthenticatedData() []byte { return []byte(c) } + +// MutableTransformer allows a transformer to be changed safely at runtime. +type MutableTransformer struct { + lock sync.RWMutex + transformer Transformer +} + +// NewMutableTransformer creates a transformer that can be updated at any time by calling Set() +func NewMutableTransformer(transformer Transformer) *MutableTransformer { + return &MutableTransformer{transformer: transformer} +} + +// Set updates the nested transformer. +func (t *MutableTransformer) Set(transformer Transformer) { + t.lock.Lock() + t.transformer = transformer + t.lock.Unlock() +} + +func (t *MutableTransformer) TransformFromStorage(data []byte, context Context) (out []byte, stale bool, err error) { + t.lock.RLock() + transformer := t.transformer + t.lock.RUnlock() + return transformer.TransformFromStorage(data, context) +} +func (t *MutableTransformer) TransformToStorage(data []byte, context Context) (out []byte, err error) { + t.lock.RLock() + transformer := t.transformer + t.lock.RUnlock() + return transformer.TransformToStorage(data, context) +} + +// PrefixTransformer holds a transformer interface and the prefix that the transformation is located under. +type PrefixTransformer struct { + Prefix []byte + Transformer Transformer +} + +type prefixTransformers struct { + transformers []PrefixTransformer + err error +} + +var _ Transformer = &prefixTransformers{} + +// NewPrefixTransformers supports the Transformer interface by checking the incoming data against the provided +// prefixes in order. The first matching prefix will be used to transform the value (the prefix is stripped +// before the Transformer interface is invoked). The first provided transformer will be used when writing to +// the store. +func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transformer { + if err == nil { + err = fmt.Errorf("the provided value does not match any of the supported transformers") + } + return &prefixTransformers{ + transformers: transformers, + err: err, + } +} + +// TransformFromStorage finds the first transformer with a prefix matching the provided data and returns +// the result of transforming the value. It will always mark any transformation as stale that is not using +// the first transformer. +func (t *prefixTransformers) TransformFromStorage(data []byte, context Context) ([]byte, bool, error) { + for i, transformer := range t.transformers { + if bytes.HasPrefix(data, transformer.Prefix) { + result, stale, err := transformer.Transformer.TransformFromStorage(data[len(transformer.Prefix):], context) + return result, stale || i != 0, err + } + } + return nil, false, t.err +} + +// TransformToStorage uses the first transformer and adds its prefix to the data. +func (t *prefixTransformers) TransformToStorage(data []byte, context Context) ([]byte, error) { + transformer := t.transformers[0] + prefixedData := make([]byte, len(transformer.Prefix), len(data)+len(transformer.Prefix)) + copy(prefixedData, transformer.Prefix) + result, err := transformer.Transformer.TransformToStorage(data, context) + if err != nil { + return nil, err + } + prefixedData = append(prefixedData, result...) + return prefixedData, nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go new file mode 100644 index 000000000000..63b9e0a75430 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/transformer_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value + +import ( + "bytes" + "fmt" + "testing" +) + +type testTransformer struct { + from, to []byte + err error + stale bool + receivedFrom, receivedTo []byte +} + +func (t *testTransformer) TransformFromStorage(from []byte, context Context) (data []byte, stale bool, err error) { + t.receivedFrom = from + return t.from, t.stale, t.err +} + +func (t *testTransformer) TransformToStorage(to []byte, context Context) (data []byte, err error) { + t.receivedTo = to + return t.to, t.err +} + +func TestPrefixFrom(t *testing.T) { + testErr := fmt.Errorf("test error") + transformErr := fmt.Errorf("test error") + transformers := []PrefixTransformer{ + {Prefix: []byte("first:"), Transformer: &testTransformer{from: []byte("value1")}}, + {Prefix: []byte("second:"), Transformer: &testTransformer{from: []byte("value2")}}, + {Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}}, + {Prefix: []byte("stale:"), Transformer: &testTransformer{from: []byte("value3"), stale: true}}, + } + p := NewPrefixTransformers(testErr, transformers...) + + testCases := []struct { + input []byte + expect []byte + stale bool + err error + match int + }{ + {[]byte("first:value"), []byte("value1"), false, nil, 0}, + {[]byte("second:value"), []byte("value2"), true, nil, 1}, + {[]byte("third:value"), nil, false, testErr, -1}, + {[]byte("fails:value"), nil, true, transformErr, 2}, + {[]byte("stale:value"), []byte("value3"), true, nil, 3}, + } + for i, test := range testCases { + got, stale, err := p.TransformFromStorage(test.input, nil) + if err != test.err || stale != test.stale || !bytes.Equal(got, test.expect) { + t.Errorf("%d: unexpected out: %q %t %#v", i, string(got), stale, err) + continue + } + if test.match != -1 && !bytes.Equal([]byte("value"), transformers[test.match].Transformer.(*testTransformer).receivedFrom) { + t.Errorf("%d: unexpected value received by transformer: %s", i, transformers[test.match].Transformer.(*testTransformer).receivedFrom) + } + } +} + +func TestPrefixTo(t *testing.T) { + testErr := fmt.Errorf("test error") + transformErr := fmt.Errorf("test error") + testCases := []struct { + transformers []PrefixTransformer + expect []byte + err error + }{ + {[]PrefixTransformer{{Prefix: []byte("first:"), Transformer: &testTransformer{to: []byte("value1")}}}, []byte("first:value1"), nil}, + {[]PrefixTransformer{{Prefix: []byte("second:"), Transformer: &testTransformer{to: []byte("value2")}}}, []byte("second:value2"), nil}, + {[]PrefixTransformer{{Prefix: []byte("fails:"), Transformer: &testTransformer{err: transformErr}}}, nil, transformErr}, + } + for i, test := range testCases { + p := NewPrefixTransformers(testErr, test.transformers...) + got, err := p.TransformToStorage([]byte("value"), nil) + if err != test.err || !bytes.Equal(got, test.expect) { + t.Errorf("%d: unexpected out: %q %#v", i, string(got), err) + continue + } + if !bytes.Equal([]byte("value"), test.transformers[0].Transformer.(*testTransformer).receivedTo) { + t.Errorf("%d: unexpected value received by transformer: %s", i, test.transformers[0].Transformer.(*testTransformer).receivedTo) + } + } +} From a73990a33f95713f026ee7ae9ae6741255aaf8e4 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 22 Feb 2017 19:22:37 -0500 Subject: [PATCH 2/4] Update etcd3 storage to leverage storage/value interfaces Adds context argument which must be set for AES GCM authenticated data to be passed. --- .../apiserver/pkg/storage/etcd3/store.go | 67 +++++++++---------- .../apiserver/pkg/storage/etcd3/store_test.go | 11 ++- .../apiserver/pkg/storage/etcd3/watcher.go | 9 +-- .../storage/storagebackend/factory/etcd3.go | 5 +- .../pkg/storage/tests/cacher_test.go | 3 +- 5 files changed, 52 insertions(+), 43 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go index c138899e3e60..1e9fbed8ed13 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store.go @@ -36,27 +36,25 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd" + "k8s.io/apiserver/pkg/storage/value" utiltrace "k8s.io/apiserver/pkg/util/trace" ) -// ValueTransformer allows a string value to be transformed before being read from or written to the underlying store. The methods -// must be able to undo the transformation caused by the other. -type ValueTransformer interface { - // TransformFromStorage may transform the provided data from its underlying storage representation or return an error. - // Stale is true if the object on disk is stale and a write to etcd should be issued, even if the contents of the object - // have not changed. - TransformFromStorage([]byte) (data []byte, stale bool, err error) - // TransformToStorage may transform the provided data into the appropriate form in storage or return an error. - TransformToStorage([]byte) (data []byte, err error) +// authenticatedDataString satisfies the value.Context interface. It uses the key to +// authenticate the stored data. This does not defend against reuse of previously +// encrypted values under the same key, but will prevent an attacker from using an +// encrypted value from a different key. A stronger authenticated data segment would +// include the etcd3 Version field (which is incremented on each write to a key and +// reset when the key is deleted), but an attacker with write access to etcd can +// force deletion and recreation of keys to weaken that angle. +type authenticatedDataString string + +// AuthenticatedData implements the value.Context interface. +func (d authenticatedDataString) AuthenticatedData() []byte { + return []byte(string(d)) } -type identityTransformer struct{} - -func (identityTransformer) TransformFromStorage(b []byte) ([]byte, bool, error) { return b, false, nil } -func (identityTransformer) TransformToStorage(b []byte) ([]byte, error) { return b, nil } - -// IdentityTransformer performs no transformation on the provided values. -var IdentityTransformer ValueTransformer = identityTransformer{} +var _ value.Context = authenticatedDataString("") type store struct { client *clientv3.Client @@ -65,7 +63,7 @@ type store struct { getOps []clientv3.OpOption codec runtime.Codec versioner storage.Versioner - transformer ValueTransformer + transformer value.Transformer pathPrefix string watcher *watcher } @@ -84,17 +82,17 @@ type objState struct { } // New returns an etcd3 implementation of storage.Interface. -func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer ValueTransformer) storage.Interface { +func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer value.Transformer) storage.Interface { return newStore(c, true, codec, prefix, transformer) } // NewWithNoQuorumRead returns etcd3 implementation of storage.Interface // where Get operations don't require quorum read. -func NewWithNoQuorumRead(c *clientv3.Client, codec runtime.Codec, prefix string, transformer ValueTransformer) storage.Interface { +func NewWithNoQuorumRead(c *clientv3.Client, codec runtime.Codec, prefix string, transformer value.Transformer) storage.Interface { return newStore(c, false, codec, prefix, transformer) } -func newStore(c *clientv3.Client, quorumRead bool, codec runtime.Codec, prefix string, transformer ValueTransformer) *store { +func newStore(c *clientv3.Client, quorumRead bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store { versioner := etcd.APIObjectVersioner{} result := &store{ client: c, @@ -136,7 +134,7 @@ func (s *store) Get(ctx context.Context, key string, resourceVersion string, out } kv := getResp.Kvs[0] - data, _, err := s.transformer.TransformFromStorage(kv.Value) + data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key)) if err != nil { return storage.NewInternalError(err.Error()) } @@ -160,7 +158,7 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, return err } - newData, err := s.transformer.TransformToStorage(data) + newData, err := s.transformer.TransformToStorage(data, authenticatedDataString(key)) if err != nil { return storage.NewInternalError(err.Error()) } @@ -185,16 +183,16 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, } // Delete implements storage.Interface.Delete. -func (s *store) Delete(ctx context.Context, key string, out runtime.Object, precondtions *storage.Preconditions) error { +func (s *store) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions) error { v, err := conversion.EnforcePtr(out) if err != nil { panic("unable to convert output object to pointer") } key = path.Join(s.pathPrefix, key) - if precondtions == nil { + if preconditions == nil { return s.unconditionalDelete(ctx, key, out) } - return s.conditionalDelete(ctx, key, out, v, precondtions) + return s.conditionalDelete(ctx, key, out, v, preconditions) } func (s *store) unconditionalDelete(ctx context.Context, key string, out runtime.Object) error { @@ -213,14 +211,14 @@ func (s *store) unconditionalDelete(ctx context.Context, key string, out runtime } kv := getResp.Kvs[0] - data, _, err := s.transformer.TransformFromStorage(kv.Value) + data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key)) if err != nil { return storage.NewInternalError(err.Error()) } return decode(s.codec, s.versioner, data, out, kv.ModRevision) } -func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.Object, v reflect.Value, precondtions *storage.Preconditions) error { +func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.Object, v reflect.Value, preconditions *storage.Preconditions) error { getResp, err := s.client.KV.Get(ctx, key) if err != nil { return err @@ -230,7 +228,7 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O if err != nil { return err } - if err := checkPreconditions(key, precondtions, origState.obj); err != nil { + if err := checkPreconditions(key, preconditions, origState.obj); err != nil { return err } txnResp, err := s.client.KV.Txn(ctx).If( @@ -255,7 +253,7 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O // GuaranteedUpdate implements storage.Interface.GuaranteedUpdate. func (s *store) GuaranteedUpdate( ctx context.Context, key string, out runtime.Object, ignoreNotFound bool, - precondtions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error { + preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error { trace := utiltrace.New(fmt.Sprintf("GuaranteedUpdate etcd3: %s", reflect.TypeOf(out).String())) defer trace.LogIfLong(500 * time.Millisecond) @@ -283,8 +281,9 @@ func (s *store) GuaranteedUpdate( } trace.Step("initial value restored") + transformContext := authenticatedDataString(key) for { - if err := checkPreconditions(key, precondtions, origState.obj); err != nil { + if err := checkPreconditions(key, preconditions, origState.obj); err != nil { return err } @@ -301,7 +300,7 @@ func (s *store) GuaranteedUpdate( return decode(s.codec, s.versioner, origState.data, out, origState.rev) } - newData, err := s.transformer.TransformToStorage(data) + newData, err := s.transformer.TransformToStorage(data, transformContext) if err != nil { return storage.NewInternalError(err.Error()) } @@ -354,7 +353,7 @@ func (s *store) GetToList(ctx context.Context, key string, resourceVersion strin if len(getResp.Kvs) == 0 { return nil } - data, _, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value) + data, _, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value, authenticatedDataString(key)) if err != nil { return storage.NewInternalError(err.Error()) } @@ -389,7 +388,7 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor elems := make([]*elemForDecode, 0, len(getResp.Kvs)) for _, kv := range getResp.Kvs { - data, _, err := s.transformer.TransformFromStorage(kv.Value) + data, _, err := s.transformer.TransformFromStorage(kv.Value, authenticatedDataString(key)) if err != nil { utilruntime.HandleError(fmt.Errorf("unable to transform key %q: %v", key, err)) continue @@ -439,7 +438,7 @@ func (s *store) getState(getResp *clientv3.GetResponse, key string, v reflect.Va return nil, err } } else { - data, stale, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value) + data, stale, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value, authenticatedDataString(key)) if err != nil { return nil, storage.NewInternalError(err.Error()) } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go index f6a335b060a5..77f397e80468 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/store_test.go @@ -35,6 +35,7 @@ import ( examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/storage" storagetests "k8s.io/apiserver/pkg/storage/tests" + "k8s.io/apiserver/pkg/storage/value" "github.com/coreos/etcd/integration" "golang.org/x/net/context" @@ -56,13 +57,19 @@ type prefixTransformer struct { err error } -func (p prefixTransformer) TransformFromStorage(b []byte) ([]byte, bool, error) { +func (p prefixTransformer) TransformFromStorage(b []byte, ctx value.Context) ([]byte, bool, error) { + if ctx == nil { + panic("no context provided") + } if !bytes.HasPrefix(b, p.prefix) { return nil, false, fmt.Errorf("value does not have expected prefix: %s", string(b)) } return bytes.TrimPrefix(b, p.prefix), p.stale, p.err } -func (p prefixTransformer) TransformToStorage(b []byte) ([]byte, error) { +func (p prefixTransformer) TransformToStorage(b []byte, ctx value.Context) ([]byte, error) { + if ctx == nil { + panic("no context provided") + } if len(b) > 0 { return append(append([]byte{}, p.prefix...), b...), p.err } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go index fca49d9cccdb..b42ae4b80f41 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/storage" + "k8s.io/apiserver/pkg/storage/value" "github.com/coreos/etcd/clientv3" etcdrpc "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" @@ -43,7 +44,7 @@ type watcher struct { client *clientv3.Client codec runtime.Codec versioner storage.Versioner - transformer ValueTransformer + transformer value.Transformer } // watchChan implements watch.Interface. @@ -60,7 +61,7 @@ type watchChan struct { errChan chan error } -func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.Versioner, transformer ValueTransformer) *watcher { +func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.Versioner, transformer value.Transformer) *watcher { return &watcher{ client: client, codec: codec, @@ -343,7 +344,7 @@ func (wc *watchChan) sendEvent(e *event) { func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) { if !e.isDeleted { - data, _, err := wc.watcher.transformer.TransformFromStorage(e.value) + data, _, err := wc.watcher.transformer.TransformFromStorage(e.value, authenticatedDataString(e.key)) if err != nil { return nil, nil, err } @@ -358,7 +359,7 @@ func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtim // we need the object only to compute whether it was filtered out // before). if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) { - data, _, err := wc.watcher.transformer.TransformFromStorage(e.prevValue) + data, _, err := wc.watcher.transformer.TransformFromStorage(e.prevValue, authenticatedDataString(e.key)) if err != nil { return nil, nil, err } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index b7b3b799c40e..d0da84cded17 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -20,6 +20,7 @@ import ( "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" + "k8s.io/apiserver/pkg/storage/value" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" @@ -56,7 +57,7 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e client.Close() } if c.Quorum { - return etcd3.New(client, c.Codec, c.Prefix, etcd3.IdentityTransformer), destroyFunc, nil + return etcd3.New(client, c.Codec, c.Prefix, value.IdentityTransformer), destroyFunc, nil } - return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, etcd3.IdentityTransformer), destroyFunc, nil + return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, value.IdentityTransformer), destroyFunc, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go b/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go index 213e1e4e72b6..619c3ea83685 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/tests/cacher_test.go @@ -42,6 +42,7 @@ import ( "k8s.io/apiserver/pkg/storage/etcd/etcdtest" etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" "k8s.io/apiserver/pkg/storage/etcd3" + "k8s.io/apiserver/pkg/storage/value" "golang.org/x/net/context" @@ -92,7 +93,7 @@ func AddObjectMetaFieldsSet(source fields.Set, objectMeta *metav1.ObjectMeta, ha func newEtcdTestStorage(t *testing.T, prefix string) (*etcdtesting.EtcdTestServer, storage.Interface) { server, _ := etcdtesting.NewUnsecuredEtcd3TestClientServer(t, scheme) - storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion), prefix, etcd3.IdentityTransformer) + storage := etcd3.New(server.V3Client, apitesting.TestCodec(codecs, examplev1.SchemeGroupVersion), prefix, value.IdentityTransformer) return server, storage } From 4f27d8feea0cbb7634554ef8af1ad6265738a2a1 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 22 Feb 2017 19:28:53 -0500 Subject: [PATCH 3/4] Allow config to accept a Transformer --- .../pkg/storage/storagebackend/config.go | 7 ++++++- .../pkg/storage/storagebackend/factory/etcd3.go | 16 ++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go index dc55bc42870a..2a88ddf74530 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/config.go @@ -16,7 +16,10 @@ limitations under the License. package storagebackend -import "k8s.io/apimachinery/pkg/runtime" +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/storage/value" +) const ( StorageTypeUnset = "" @@ -45,6 +48,8 @@ type Config struct { Codec runtime.Codec Copier runtime.ObjectCopier + // Transformer allows the value to be transformed prior to persisting into etcd. + Transformer value.Transformer } func NewDefaultConfig(prefix string, copier runtime.ObjectCopier, codec runtime.Codec) *Config { diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index d0da84cded17..2c838a09ce74 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -17,14 +17,14 @@ limitations under the License. package factory import ( + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/pkg/transport" + "golang.org/x/net/context" + "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/storage/value" - - "github.com/coreos/etcd/clientv3" - "github.com/coreos/etcd/pkg/transport" - "golang.org/x/net/context" ) func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) { @@ -56,8 +56,12 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e cancel() client.Close() } + transformer := c.Transformer + if transformer == nil { + transformer = value.IdentityTransformer + } if c.Quorum { - return etcd3.New(client, c.Codec, c.Prefix, value.IdentityTransformer), destroyFunc, nil + return etcd3.New(client, c.Codec, c.Prefix, transformer), destroyFunc, nil } - return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, value.IdentityTransformer), destroyFunc, nil + return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, transformer), destroyFunc, nil } From 7827899b1dd22074eb230e241f63e69499046fb6 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Wed, 22 Feb 2017 19:29:32 -0500 Subject: [PATCH 4/4] generated: bazel --- hack/.linted_packages | 1 + .../k8s.io/apiserver/pkg/storage/etcd3/BUILD | 2 ++ .../pkg/storage/storagebackend/BUILD | 5 +++- .../pkg/storage/storagebackend/factory/BUILD | 1 + .../k8s.io/apiserver/pkg/storage/tests/BUILD | 1 + .../k8s.io/apiserver/pkg/storage/value/BUILD | 22 +++++++++++++++++ .../pkg/storage/value/encrypt/aes/BUILD | 24 +++++++++++++++++++ 7 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/BUILD create mode 100644 staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/BUILD diff --git a/hack/.linted_packages b/hack/.linted_packages index 0be943a2a6d5..6a027c8a7e6c 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -333,6 +333,7 @@ staging/src/k8s.io/apiserver/pkg/storage/etcd3 staging/src/k8s.io/apiserver/pkg/storage/names staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory +staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes staging/src/k8s.io/apiserver/pkg/util/flushwriter staging/src/k8s.io/apiserver/pkg/util/logs staging/src/k8s.io/apiserver/plugin/pkg/authenticator diff --git a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD index 5e2b782d6004..41f204d1f3ff 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/BUILD @@ -36,6 +36,7 @@ go_test( "//vendor/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/tests:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", ], ) @@ -62,6 +63,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/etcd:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/trace:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD index 831f7ece28c9..be0b157596e7 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/BUILD @@ -11,5 +11,8 @@ go_library( name = "go_default_library", srcs = ["config.go"], tags = ["automanaged"], - deps = ["//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library"], + deps = [ + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", + ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD index a288ae050e82..52bcd7d6c0f7 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/BUILD @@ -46,5 +46,6 @@ go_library( "//vendor/k8s.io/apiserver/pkg/storage/etcd:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD index eede87f675b0..17d4a8dedc64 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/tests/BUILD @@ -34,6 +34,7 @@ go_test( "//vendor/k8s.io/apiserver/pkg/storage/etcd/etcdtest:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/etcd3:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library", ], ) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD new file mode 100644 index 000000000000..ac114674283a --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/BUILD @@ -0,0 +1,22 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["transformer_test.go"], + library = ":go_default_library", + tags = ["automanaged"], +) + +go_library( + name = "go_default_library", + srcs = ["transformer.go"], + tags = ["automanaged"], +) diff --git a/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/BUILD new file mode 100644 index 000000000000..401c570ba93f --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/storage/value/encrypt/aes/BUILD @@ -0,0 +1,24 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["aes_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = ["//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library"], +) + +go_library( + name = "go_default_library", + srcs = ["aes.go"], + tags = ["automanaged"], + deps = ["//vendor/k8s.io/apiserver/pkg/storage/value:go_default_library"], +)