From 21e1131a303b928b3dfa0b835f6f650c868217bc Mon Sep 17 00:00:00 2001 From: Jerome Froelich Date: Mon, 15 May 2017 12:09:18 -0400 Subject: [PATCH] First pass at logic for encoding policies in a bit flag --- glide.lock | 6 +- glide.yaml | 2 + policy/hpack.go | 191 +++++++++++++++++++++++++ policy/hpack_test.go | 242 ++++++++++++++++++++++++++++++++ protocol/msgpack/wire_format.md | 1 + 5 files changed, 440 insertions(+), 2 deletions(-) create mode 100644 policy/hpack.go create mode 100644 policy/hpack_test.go diff --git a/glide.lock b/glide.lock index 7c6afb7..e0a700d 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 07a1a24c24370b50b8cb4f21bdd6fd9b1b787f00c87b1f3f95715f93d84594f2 -updated: 2017-05-09T23:04:21.133965573-04:00 +hash: bc258b529b8275653fa881938866343a45005dfc7cb523636a7445d2dfb4de27 +updated: 2017-05-15T17:39:01.10730566-04:00 imports: - name: github.com/apache/thrift version: 9549b25c77587b29be4e0b5c258221a4ed85d37a @@ -41,6 +41,8 @@ imports: version: bb4de0191aa41b5507caa14b0650cdbddcd9280b - name: github.com/StackExchange/wmi version: e542ed97d15e640bdc14b5c12162d59e8fc67324 +- name: github.com/tmthrgd/go-popcount + version: 9e009648b013fc84ef2e69fdc5264dc498ac3391 - name: github.com/uber-go/atomic version: e682c1008ac17bf26d2e4b5ad6cdd08520ed0b22 - name: github.com/uber-go/tally diff --git a/glide.yaml b/glide.yaml index 91c346e..5f613b8 100644 --- a/glide.yaml +++ b/glide.yaml @@ -53,6 +53,8 @@ import: - difflib - package: gopkg.in/yaml.v2 version: a83829b6f1293c91addabc89d0571c246397bbf4 +- package: github.com/tmthrgd/go-popcount + version: 9e009648b013fc84ef2e69fdc5264dc498ac3391 testImport: - package: github.com/stretchr/testify version: d77da356e56a7428ad25149ca77381849a6a5232 diff --git a/policy/hpack.go b/policy/hpack.go new file mode 100644 index 0000000..042fda3 --- /dev/null +++ b/policy/hpack.go @@ -0,0 +1,191 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package policy + +import ( + "fmt" + + "github.com/tmthrgd/go-popcount" +) + +const ( + // NB(jeromefroe): Since PoliciesFlag can represent up to 63 unique + // policies, the default size of the dynamic tables should be 63 as well. + defaultTableSize = 63 +) + +// Bitflag is a bitflag representation of a slice of policies. It can +// represent up to 63 unique policies or any combination thereof. +type Bitflag uint64 + +// EmptyBitflag is a bitflag that represents an empty slice of policies. +var EmptyBitflag Bitflag + +// NewBitflag returns a new empty bitflag. +func NewBitflag() Bitflag { return EmptyBitflag } + +// Set sets the nth bit of a PoliciesFlag. It panics if n > 63. +func (i Bitflag) Set(n uint) Bitflag { + if n > 63 { + err := fmt.Errorf( + "tried to set %vth bit of a PoliciesFlag, only bits 0-63 can be set", + n, + ) + panic(err) + } + + i = i | (1 << n) + return i +} + +// Iterator returns an iterator for a policies bitflag. It makes a copy +// of the flag so that the original bitflag can be safely mutated. +func (i Bitflag) Iterator() BitflagIterator { + return BitflagIterator{ + flag: i, + idx: -1, + } +} + +// BitflagIterator is an iterator for iterating over the bits in a +// policies bitflag. +type BitflagIterator struct { + flag Bitflag + idx int +} + +// Next returns a boolean indicating whether the policies bitflag has any +// remaining bits set. +func (i *BitflagIterator) Next() bool { + var set bool + for { + if i.flag == 0 { + return false + } + i.idx++ + + set = (i.flag & (1 << uint(i.idx))) > 0 + if set { + // Unset the current bit. + i.flag = i.flag ^ (1 << uint(i.idx)) + return true + } + } +} + +// Index returns the index of the currently set bit in the policies bitflag. +func (i BitflagIterator) Index() int { + return i.idx +} + +// Encoder encodes policies into a bitflag. +type Encoder struct { + dynamicTable map[Policy]uint +} + +// NewEncoder returns a new Encoder. +func NewEncoder() Encoder { + dt := make(map[Policy]uint, defaultTableSize) + return Encoder{dt} +} + +// Encode encodes a slice of polices into a bitflag. It returns a bitflag +// representation of the policies it encoded and a slice of policies that +// it could not encode because they are new. New policies are inserted into +// its table so that subsequent calls to Encode will be able to encode them +// in the returned bitflag as well. It accepts a buffer argument so that +// slices of policies can be reused between calls. +func (e *Encoder) Encode(policies, buffer []Policy) (Bitflag, []Policy) { + var flag Bitflag + if buffer == nil { + buffer = make([]Policy, 0) + } + + for _, policy := range policies { + id, ok := e.dynamicTable[policy] + if !ok && len(e.dynamicTable) <= defaultTableSize { + e.dynamicTable[policy] = uint(len(e.dynamicTable)) + buffer = append(buffer, policy) + continue + } + + flag = flag.Set(id) + } + + return flag, buffer +} + +// Decoder decodes a bitflag representing a slice of policies. +type Decoder struct { + dynamicTable []Policy +} + +// NewDecoder returns a new decoder for policy bitflags. +func NewDecoder() Decoder { + dt := make([]Policy, 0, defaultTableSize) + return Decoder{dt} +} + +// Decode decodes a policies bitflag, appending the associated policies +// to the slice of policies passed to it. Any policies in the slice passed +// to it are added to its table so they can be decoded in subsequent calls +// to Decode. Decode also returns a bitflag representing all the policies +// which were passed into it, including those in both the policies bitflag +// and the slice of policies. +func (d *Decoder) Decode( + policies []Policy, + flag Bitflag, +) (Bitflag, []Policy, error) { + if policies == nil { + l := popcount.Count64(uint64(flag)) + policies = make([]Policy, 0, l) + } + originalLen := len(policies) + + iter := flag.Iterator() + for iter.Next() { + idx := iter.Index() + if idx >= len(d.dynamicTable) { + err := fmt.Errorf( + "encountered invalid index decoding bitflag, %v was set but the decoder only has %v entries", + idx, + len(d.dynamicTable), + ) + return 0, policies, err + } + + policies = append(policies, d.dynamicTable[idx]) + } + + for i, policy := range policies { + if i == originalLen { + break + } + + if len(d.dynamicTable) < defaultTableSize { + idx := uint(len(d.dynamicTable)) + flag = flag.Set(idx) + d.dynamicTable = append(d.dynamicTable, policy) + } + } + + return flag, policies, nil +} diff --git a/policy/hpack_test.go b/policy/hpack_test.go new file mode 100644 index 0000000..f223fe9 --- /dev/null +++ b/policy/hpack_test.go @@ -0,0 +1,242 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package policy + +import ( + "testing" + "time" + + "github.com/m3db/m3x/time" + "github.com/stretchr/testify/require" +) + +func TestBitflag(t *testing.T) { + bf := NewBitflag().Set(0). + Set(1). + Set(4) + + var expected uint64 = 19 + require.Equal(t, expected, uint64(bf)) + + require.Panics(t, func() { + bf.Set(64) + }) +} + +func TestBitflagIterator(t *testing.T) { + bf := NewBitflag().Set(0). + Set(1). + Set(4). + Set(11). + Set(37) + iter := bf.Iterator() + + tests := []struct { + expectedNext bool + expectedIndex int + }{ + {true, 0}, + {true, 1}, + {true, 4}, + {true, 11}, + {true, 37}, + {false, -1}, + } + + for _, test := range tests { + next := iter.Next() + require.Equal(t, test.expectedNext, next) + if next { + require.Equal(t, test.expectedIndex, iter.Index()) + } + } +} + +func TestEncoder(t *testing.T) { + var ( + enc = NewEncoder() + firstPolicy = NewPolicy(time.Second, xtime.Second, time.Hour) + secondPolicy = NewPolicy(10*time.Second, xtime.Second, 24*time.Hour) + thirdPolicy = NewPolicy(time.Minute, xtime.Minute, 24*7*time.Hour) + buffer = make([]Policy, 0) + ) + + tests := []struct { + policies, expectedPolicies []Policy + expectedBitflag Bitflag + }{ + { + // New policies should be returned in the slice of policies. + policies: []Policy{firstPolicy, secondPolicy}, + expectedPolicies: []Policy{firstPolicy, secondPolicy}, + expectedBitflag: NewBitflag(), + }, + { + // Since these policies have been seen already they should + // now be encoded in the bitflag. + policies: []Policy{firstPolicy, secondPolicy}, + expectedPolicies: []Policy{}, + expectedBitflag: NewBitflag().Set(0).Set(1), + }, + { + // The old policies should still be encoded in the bitflag but + // the new policy should be returned in the slice of policies. + policies: []Policy{firstPolicy, secondPolicy, thirdPolicy}, + expectedPolicies: []Policy{thirdPolicy}, + expectedBitflag: NewBitflag().Set(0).Set(1), + }, + { + // All policies have been seen at least once before and so they + // should all be encoded in the bitflag. + policies: []Policy{firstPolicy, secondPolicy, thirdPolicy}, + expectedPolicies: []Policy{}, + expectedBitflag: NewBitflag().Set(0).Set(1).Set(2), + }, + { + // Removed policies should not be included in the returned bitflag. + policies: []Policy{firstPolicy, thirdPolicy}, + expectedPolicies: []Policy{}, + expectedBitflag: NewBitflag().Set(0).Set(2), + }, + } + + for _, test := range tests { + buffer = buffer[:0] + actualBitflag, actualPolicies := enc.Encode(test.policies, buffer) + require.Equal(t, test.expectedBitflag, actualBitflag) + require.Equal(t, test.expectedPolicies, actualPolicies) + } +} + +func TestDecoder(t *testing.T) { + var ( + dec = NewDecoder() + firstPolicy = NewPolicy(time.Second, xtime.Second, time.Hour) + secondPolicy = NewPolicy(10*time.Second, xtime.Second, 24*time.Hour) + thirdPolicy = NewPolicy(time.Minute, xtime.Minute, 24*7*time.Hour) + ) + + tests := []struct { + policies, expectedPolicies []Policy + bitflag, expectedBitflag Bitflag + }{ + { + // New policies should be added to returned bitflag. + policies: []Policy{firstPolicy, secondPolicy}, + expectedPolicies: []Policy{firstPolicy, secondPolicy}, + bitflag: NewBitflag(), + expectedBitflag: NewBitflag().Set(0).Set(1), + }, + { + // Policies that were seen previously should be decoded from + // the bitflag. + policies: []Policy{}, + expectedPolicies: []Policy{firstPolicy, secondPolicy}, + bitflag: NewBitflag().Set(0).Set(1), + expectedBitflag: NewBitflag().Set(0).Set(1), + }, + { + // New policies should be returned in the bitflag in addition + // to any old policies. + policies: []Policy{thirdPolicy}, + expectedPolicies: []Policy{thirdPolicy, firstPolicy, secondPolicy}, + bitflag: NewBitflag().Set(0).Set(1), + expectedBitflag: NewBitflag().Set(0).Set(1).Set(2), + }, + { + // All policies should now be decoded from the bitflag. + policies: []Policy{}, + expectedPolicies: []Policy{firstPolicy, secondPolicy, thirdPolicy}, + bitflag: NewBitflag().Set(0).Set(1).Set(2), + expectedBitflag: NewBitflag().Set(0).Set(1).Set(2), + }, + { + // Removed policies should not be returned. + policies: []Policy{}, + expectedPolicies: []Policy{firstPolicy, thirdPolicy}, + bitflag: NewBitflag().Set(0).Set(2), + expectedBitflag: NewBitflag().Set(0).Set(2), + }, + } + + for _, test := range tests { + actualBitflag, actualPolicies, err := dec.Decode(test.policies, test.bitflag) + require.NoError(t, err) + require.Equal(t, test.expectedBitflag, actualBitflag) + require.Equal(t, test.expectedPolicies, actualPolicies) + } +} + +func TestDecodeError(t *testing.T) { + var ( + dec = NewDecoder() + policies = make([]Policy, 0) + bitflag = NewBitflag().Set(0) + ) + + // Decoder should return an error when encountering a bitflag it has + // never seen before. + _, _, err := dec.Decode(policies, bitflag) + require.Error(t, err) +} + +func TestRoundTrip(t *testing.T) { + var ( + enc = NewEncoder() + dec = NewDecoder() + firstPolicy = NewPolicy(time.Second, xtime.Second, time.Hour) + secondPolicy = NewPolicy(10*time.Second, xtime.Second, 24*time.Hour) + thirdPolicy = NewPolicy(time.Minute, xtime.Minute, 24*7*time.Hour) + buffer = make([]Policy, 0) + ) + + tests := []struct { + policies, expectedPolicies []Policy + expectedBitflag Bitflag + }{ + { + policies: []Policy{firstPolicy, secondPolicy}, + expectedPolicies: []Policy{firstPolicy, secondPolicy}, + expectedBitflag: NewBitflag().Set(0).Set(1), + }, + { + // Add a new policy. + policies: []Policy{firstPolicy, secondPolicy, thirdPolicy}, + expectedPolicies: []Policy{thirdPolicy, firstPolicy, secondPolicy}, + expectedBitflag: NewBitflag().Set(0).Set(1).Set(2), + }, + { + // Remove a policy. + policies: []Policy{firstPolicy, thirdPolicy}, + expectedPolicies: []Policy{firstPolicy, thirdPolicy}, + expectedBitflag: NewBitflag().Set(0).Set(2), + }, + } + + for _, test := range tests { + buffer = buffer[:0] + bitflag, policies := enc.Encode(test.policies, buffer) + actualBitflag, actualPolicies, err := dec.Decode(policies, bitflag) + require.NoError(t, err) + require.Equal(t, test.expectedBitflag, actualBitflag) + require.Equal(t, test.expectedPolicies, actualPolicies) + } +} diff --git a/protocol/msgpack/wire_format.md b/protocol/msgpack/wire_format.md index 908e3fc..61ee4d6 100644 --- a/protocol/msgpack/wire_format.md +++ b/protocol/msgpack/wire_format.md @@ -53,6 +53,7 @@ * Cutover * Tombstoned * List of Policy objects + * Policies Bitflag * Policy object * Number of Policy fields