diff --git a/helper/forwarding/types.pb.go b/helper/forwarding/types.pb.go index 4ed46ba405e8..d17c4f1b4d75 100644 --- a/helper/forwarding/types.pb.go +++ b/helper/forwarding/types.pb.go @@ -23,7 +23,7 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type Request struct { // Not used right now but reserving in case it turns out that streaming // makes things more economical on the gRPC side - // uint64 id = 1; + //uint64 id = 1; Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"` Url *URL `protobuf:"bytes,3,opt,name=url,proto3" json:"url,omitempty"` HeaderEntries map[string]*HeaderEntry `protobuf:"bytes,4,rep,name=header_entries,json=headerEntries,proto3" json:"header_entries,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -115,12 +115,12 @@ type URL struct { Opaque string `protobuf:"bytes,2,opt,name=opaque,proto3" json:"opaque,omitempty"` // This isn't needed now but might be in the future, so we'll skip the // number to keep the ordering in net/url - // UserInfo user = 3; + //UserInfo user = 3; Host string `protobuf:"bytes,4,opt,name=host,proto3" json:"host,omitempty"` Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` RawPath string `protobuf:"bytes,6,opt,name=raw_path,json=rawPath,proto3" json:"raw_path,omitempty"` // This also isn't needed right now, but we'll reserve the number - // bool force_query = 7; + //bool force_query = 7; RawQuery string `protobuf:"bytes,8,opt,name=raw_query,json=rawQuery,proto3" json:"raw_query,omitempty"` Fragment string `protobuf:"bytes,9,opt,name=fragment,proto3" json:"fragment,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -244,7 +244,7 @@ func (m *HeaderEntry) GetValues() []string { type Response struct { // Not used right now but reserving in case it turns out that streaming // makes things more economical on the gRPC side - // uint64 id = 1; + //uint64 id = 1; StatusCode uint32 `protobuf:"varint,2,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` Body []byte `protobuf:"bytes,3,opt,name=body,proto3" json:"body,omitempty"` // Added in 0.6.2 to ensure that the content-type is set appropriately, as diff --git a/helper/storagepacker/storagepacker_v2.go b/helper/storagepacker/storagepacker_v2.go index 6fba73f6443d..8630045ea448 100644 --- a/helper/storagepacker/storagepacker_v2.go +++ b/helper/storagepacker/storagepacker_v2.go @@ -12,7 +12,6 @@ import ( radix "github.com/armon/go-radix" "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" "github.com/hashicorp/errwrap" log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" @@ -312,7 +311,7 @@ func (s *StoragePackerV2) shardBucket(ctx context.Context, bucket *LockedBucket, LockEntry: lock, Bucket: &Bucket{ Key: fmt.Sprintf("%s/%s", bucket.Key, shardKey), - ItemMap: make(map[string]*any.Any), + ItemMap: make(map[string][]byte), }, } shards[shardKey] = shardedBucket @@ -472,12 +471,12 @@ func (s *LockedBucket) upsert(item *Item) error { } if s.ItemMap == nil { - s.ItemMap = make(map[string]*any.Any) + s.ItemMap = make(map[string][]byte) } itemHash := GetItemIDHash(item.ID) - s.ItemMap[itemHash] = item.Message + s.ItemMap[itemHash] = item.Data return nil } @@ -591,14 +590,14 @@ func (s *StoragePackerV2) GetItem(ctx context.Context, itemID string) (*Item, er itemHash := GetItemIDHash(itemID) - item, ok := bucket.ItemMap[itemHash] + data, ok := bucket.ItemMap[itemHash] if !ok { return nil, nil } return &Item{ - ID: itemID, - Message: item, + ID: itemID, + Data: data, }, nil } @@ -612,6 +611,14 @@ func (s *StoragePackerV2) PutItem(ctx context.Context, item *Item) error { return fmt.Errorf("missing ID in item") } + if item.Data == nil { + return fmt.Errorf("missing data in item") + } + + if item.Message != nil { + return fmt.Errorf("'Message' is deprecated; use 'Data' instead") + } + // Get the bucket key bucketKey := s.BucketStorageKeyForItemID(item.ID) cacheKey := s.GetCacheKey(bucketKey) diff --git a/helper/storagepacker/types.pb.go b/helper/storagepacker/types.pb.go index 8d42ec076de5..61a94977bc4c 100644 --- a/helper/storagepacker/types.pb.go +++ b/helper/storagepacker/types.pb.go @@ -29,8 +29,11 @@ type Item struct { // described above, the caller *must not* rely on this value to be // consistent with what they passed in. ID string `sentinel:"" protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // message is the contents of the item - Message *any.Any `sentinel:"" protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + // Message holds the contents of the item + // Deprecated: Use 'Data' instead + Message *any.Any `sentinel:"" protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + // Data holds the contents of the item. Used in storage packer v2. + Data []byte `sentinel:"" protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -75,6 +78,13 @@ func (m *Item) GetMessage() *any.Any { return nil } +func (m *Item) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + // Bucket is a construct to hold multiple items within itself. This // abstraction contains multiple buckets of the same kind within itself and // shares amont them the items that get inserted. When the bucket as a whole @@ -87,10 +97,10 @@ type Bucket struct { // Items holds the items contained within this bucket. Used by v1. Items []*Item `sentinel:"" protobuf:"bytes,2,rep,name=items,proto3" json:"items,omitempty"` // ItemMap stores a mapping of item ID to message. Used by v2. - ItemMap map[string]*any.Any `sentinel:"" protobuf:"bytes,3,rep,name=item_map,json=itemMap,proto3" json:"item_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ItemMap map[string][]byte `sentinel:"" protobuf:"bytes,3,rep,name=item_map,json=itemMap,proto3" json:"item_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Bucket) Reset() { *m = Bucket{} } @@ -132,7 +142,7 @@ func (m *Bucket) GetItems() []*Item { return nil } -func (m *Bucket) GetItemMap() map[string]*any.Any { +func (m *Bucket) GetItemMap() map[string][]byte { if m != nil { return m.ItemMap } @@ -142,29 +152,30 @@ func (m *Bucket) GetItemMap() map[string]*any.Any { func init() { proto.RegisterType((*Item)(nil), "storagepacker.Item") proto.RegisterType((*Bucket)(nil), "storagepacker.Bucket") - proto.RegisterMapType((map[string]*any.Any)(nil), "storagepacker.Bucket.ItemMapEntry") + proto.RegisterMapType((map[string][]byte)(nil), "storagepacker.Bucket.ItemMapEntry") } func init() { proto.RegisterFile("helper/storagepacker/types.proto", fileDescriptor_c0e98c66c4f51b7f) } var fileDescriptor_c0e98c66c4f51b7f = []byte{ - // 276 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0xcf, 0x4b, 0xc3, 0x30, - 0x14, 0xc7, 0x69, 0xeb, 0x36, 0x7d, 0x53, 0x91, 0xe8, 0xa1, 0xee, 0x54, 0x7a, 0xaa, 0x1e, 0x12, - 0x9c, 0x17, 0x11, 0x3c, 0x38, 0x50, 0xf0, 0x20, 0x48, 0x8f, 0x5e, 0x24, 0xed, 0x9e, 0x6d, 0xe8, - 0x8f, 0x84, 0x24, 0x1d, 0xf4, 0x1f, 0xf5, 0xef, 0x91, 0x36, 0x0e, 0x9c, 0x0c, 0x6f, 0x2f, 0x7c, - 0x3f, 0xf9, 0xe4, 0x1b, 0x1e, 0x44, 0x25, 0xd6, 0x0a, 0x35, 0x33, 0x56, 0x6a, 0x5e, 0xa0, 0xe2, - 0x79, 0x85, 0x9a, 0xd9, 0x5e, 0xa1, 0xa1, 0x4a, 0x4b, 0x2b, 0xc9, 0xc9, 0x4e, 0xb4, 0xb8, 0x2c, - 0xa4, 0x2c, 0x6a, 0x64, 0x63, 0x98, 0x75, 0x9f, 0x8c, 0xb7, 0xbd, 0x23, 0xe3, 0x67, 0x38, 0x78, - 0xb1, 0xd8, 0x90, 0x53, 0xf0, 0xc5, 0x3a, 0xf4, 0x22, 0x2f, 0x39, 0x4a, 0x7d, 0xb1, 0x26, 0x14, - 0x66, 0x0d, 0x1a, 0xc3, 0x0b, 0x0c, 0xfd, 0xc8, 0x4b, 0xe6, 0xcb, 0x0b, 0xea, 0x24, 0x74, 0x2b, - 0xa1, 0x8f, 0x6d, 0x9f, 0x6e, 0xa1, 0xf8, 0xcb, 0x83, 0xe9, 0xaa, 0xcb, 0x2b, 0xb4, 0xe4, 0x0c, - 0x82, 0x0a, 0xfb, 0x1f, 0xd7, 0x30, 0x92, 0x2b, 0x98, 0x08, 0x8b, 0x8d, 0x09, 0xfd, 0x28, 0x48, - 0xe6, 0xcb, 0x73, 0xba, 0x53, 0x8f, 0x0e, 0x05, 0x52, 0x47, 0x90, 0x07, 0x38, 0x1c, 0x86, 0x8f, - 0x86, 0xab, 0x30, 0x18, 0xe9, 0xf8, 0x0f, 0xed, 0x5e, 0x19, 0x2f, 0xbd, 0x72, 0xf5, 0xd4, 0x5a, - 0xdd, 0xa7, 0x33, 0xe1, 0x4e, 0x8b, 0x37, 0x38, 0xfe, 0x1d, 0xec, 0xe9, 0x72, 0x0d, 0x93, 0x0d, - 0xaf, 0xbb, 0xff, 0xbf, 0xe5, 0x90, 0x7b, 0xff, 0xce, 0x5b, 0xdd, 0xbc, 0xb3, 0x42, 0xd8, 0xb2, - 0xcb, 0x68, 0x2e, 0x1b, 0x56, 0x72, 0x53, 0x8a, 0x5c, 0x6a, 0xc5, 0x36, 0xbc, 0xab, 0x2d, 0xdb, - 0xb7, 0x89, 0x6c, 0x3a, 0xba, 0x6e, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x46, 0x9d, 0x8a, 0xcb, - 0xa8, 0x01, 0x00, 0x00, + // 289 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x3d, 0x4f, 0xf3, 0x30, + 0x14, 0x85, 0xe5, 0xa4, 0x1f, 0xef, 0xeb, 0x16, 0x84, 0x4c, 0x87, 0xd0, 0x29, 0xea, 0x14, 0x16, + 0x5b, 0x94, 0x05, 0x55, 0x62, 0xa0, 0x12, 0x03, 0x03, 0x8b, 0xc7, 0x2e, 0xc8, 0x49, 0x2e, 0x89, + 0x95, 0x0f, 0x5b, 0xb6, 0x53, 0x29, 0xff, 0x8e, 0x9f, 0x86, 0x12, 0x53, 0x89, 0xa2, 0x6e, 0xe7, + 0xea, 0x3c, 0x3e, 0x3e, 0xf6, 0xc5, 0x71, 0x09, 0xb5, 0x06, 0xc3, 0xac, 0x53, 0x46, 0x14, 0xa0, + 0x45, 0x56, 0x81, 0x61, 0xae, 0xd7, 0x60, 0xa9, 0x36, 0xca, 0x29, 0x72, 0x75, 0x66, 0xad, 0xef, + 0x0a, 0xa5, 0x8a, 0x1a, 0xd8, 0x68, 0xa6, 0xdd, 0x27, 0x13, 0x6d, 0xef, 0xc9, 0xcd, 0x01, 0x4f, + 0xde, 0x1c, 0x34, 0xe4, 0x1a, 0x07, 0x32, 0x8f, 0x50, 0x8c, 0x92, 0xff, 0x3c, 0x90, 0x39, 0xa1, + 0x78, 0xde, 0x80, 0xb5, 0xa2, 0x80, 0x28, 0x88, 0x51, 0xb2, 0xd8, 0xae, 0xa8, 0x0f, 0xa1, 0xa7, + 0x10, 0xfa, 0xd2, 0xf6, 0xfc, 0x04, 0x11, 0x82, 0x27, 0xb9, 0x70, 0x22, 0x0a, 0x63, 0x94, 0x2c, + 0xf9, 0xa8, 0x37, 0x5f, 0x08, 0xcf, 0xf6, 0x5d, 0x56, 0x81, 0x23, 0x37, 0x38, 0xac, 0xa0, 0xff, + 0xc9, 0x1f, 0x24, 0xb9, 0xc7, 0x53, 0xe9, 0xa0, 0xb1, 0x51, 0x10, 0x87, 0xc9, 0x62, 0x7b, 0x4b, + 0xcf, 0x2a, 0xd3, 0xa1, 0x14, 0xf7, 0x04, 0x79, 0xc6, 0xff, 0x06, 0xf1, 0xd1, 0x08, 0x1d, 0x85, + 0x23, 0xbd, 0xf9, 0x43, 0xfb, 0x5b, 0xc6, 0x43, 0xef, 0x42, 0xbf, 0xb6, 0xce, 0xf4, 0x7c, 0x2e, + 0xfd, 0xb4, 0xde, 0xe1, 0xe5, 0x6f, 0xe3, 0x42, 0x97, 0x15, 0x9e, 0x1e, 0x45, 0xdd, 0xf9, 0xa7, + 0x2e, 0xb9, 0x1f, 0x76, 0xc1, 0x13, 0xda, 0x3f, 0x1c, 0x58, 0x21, 0x5d, 0xd9, 0xa5, 0x34, 0x53, + 0x0d, 0x2b, 0x85, 0x2d, 0x65, 0xa6, 0x8c, 0x66, 0x47, 0xd1, 0xd5, 0x8e, 0x5d, 0xda, 0x43, 0x3a, + 0x1b, 0x3f, 0xe8, 0xf1, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x11, 0x50, 0x1a, 0x26, 0xa6, 0x01, 0x00, + 0x00, } diff --git a/helper/storagepacker/types.proto b/helper/storagepacker/types.proto index 4edfaf4f8572..189cd8e5aa26 100644 --- a/helper/storagepacker/types.proto +++ b/helper/storagepacker/types.proto @@ -14,8 +14,11 @@ message Item { // described above, the caller *must not* rely on this value to be // consistent with what they passed in. string id = 1; - // message is the contents of the item + // Message holds the contents of the item + // Deprecated: Use 'Data' instead google.protobuf.Any message = 2; + // Data holds the contents of the item. Used in storage packer v2. + bytes data = 3; } // Bucket is a construct to hold multiple items within itself. This @@ -30,5 +33,5 @@ message Bucket { // Items holds the items contained within this bucket. Used by v1. repeated Item items = 2; // ItemMap stores a mapping of item ID to message. Used by v2. - map item_map = 3; + map item_map = 3; } diff --git a/vault/external_tests/storagepacker/storagepacker_sharding_test.go b/vault/external_tests/storagepacker/storagepacker_sharding_test.go index bfadc0508a79..5205f678956e 100644 --- a/vault/external_tests/storagepacker/storagepacker_sharding_test.go +++ b/vault/external_tests/storagepacker/storagepacker_sharding_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/proto" consulapi "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/helper/storagepacker" @@ -93,18 +93,18 @@ func TestStoragePacker_Sharding(t *testing.T) { t.Fatal(err) } - protoSecret := &pb.Secret{ + secret := &pb.Secret{ InternalData: randString, } - messageAsAny, err := ptypes.MarshalAny(protoSecret) + secretProto, err := proto.Marshal(secret) if err != nil { t.Fatal(err) } for i := 0; i < numEntries; i++ { if err := packer.PutItem(ctx, &storagepacker.Item{ - ID: fmt.Sprintf("%05d", i), - Message: messageAsAny, + ID: fmt.Sprintf("%05d", i), + Data: secretProto, }); err != nil { t.Fatal(err) } diff --git a/vault/identity_store.go b/vault/identity_store.go index 75d007c765bf..127e6dacf01c 100644 --- a/vault/identity_store.go +++ b/vault/identity_store.go @@ -177,7 +177,7 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) { items := make([]*storagepacker.Item, 0, len(bucket.Items)+len(bucket.ItemMap)) items = append(items, bucket.Items...) for id, message := range bucket.ItemMap { - items = append(items, &storagepacker.Item{ID: id, Message: message}) + items = append(items, &storagepacker.Item{ID: id, Data: message}) } for _, item := range items { entity, err := i.parseEntityFromBucketItem(ctx, item) @@ -246,7 +246,7 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) { items := make([]*storagepacker.Item, 0, len(bucket.Items)+len(bucket.ItemMap)) items = append(items, bucket.Items...) for id, message := range bucket.ItemMap { - items = append(items, &storagepacker.Item{ID: id, Message: message}) + items = append(items, &storagepacker.Item{ID: id, Data: message}) } for _, item := range items { group, err := i.parseGroupFromBucketItem(item) @@ -302,8 +302,7 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto persistNeeded := false - var entity identity.Entity - err := ptypes.UnmarshalAny(item.Message, &entity) + entity, err := i.decodeEntity(item) if err != nil { // If we encounter an error, it would mean that the format of the // entity is an older one. Try decoding using the older format and if @@ -349,7 +348,7 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto entity.BucketKey = i.entityPacker.BucketKey(entity.ID) - pN, err := parseExtraEntityFromBucket(ctx, i, &entity) + pN, err := parseExtraEntityFromBucket(ctx, i, entity) if err != nil { return nil, err } @@ -358,16 +357,11 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto } if persistNeeded && !i.core.ReplicationState().HasState(consts.ReplicationPerformanceSecondary) { - entityAsAny, err := ptypes.MarshalAny(&entity) + item, err := i.encodeEntity(entity) if err != nil { return nil, err } - item := &storagepacker.Item{ - ID: entity.ID, - Message: entityAsAny, - } - // Store the entity with new format err = i.entityPacker.PutItem(ctx, item) if err != nil { @@ -379,7 +373,7 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto entity.NamespaceID = namespace.RootNamespaceID } - return &entity, nil + return entity, nil } func (i *IdentityStore) parseGroupFromBucketItem(item *storagepacker.Item) (*identity.Group, error) { @@ -387,8 +381,7 @@ func (i *IdentityStore) parseGroupFromBucketItem(item *storagepacker.Item) (*ide return nil, fmt.Errorf("nil item") } - var group identity.Group - err := ptypes.UnmarshalAny(item.Message, &group) + group, err := i.decodeGroup(item) if err != nil { return nil, errwrap.Wrapf("failed to decode group from storage bucket item: {{err}}", err) } @@ -399,7 +392,7 @@ func (i *IdentityStore) parseGroupFromBucketItem(item *storagepacker.Item) (*ide group.BucketKey = i.groupPacker.BucketKey(group.ID) - return &group, nil + return group, nil } // entityByAliasFactors fetches the entity based on factors of alias, i.e mount diff --git a/vault/identity_store_aliases.go b/vault/identity_store_aliases.go index 97cd9a5cce38..e5944780c90a 100644 --- a/vault/identity_store_aliases.go +++ b/vault/identity_store_aliases.go @@ -8,7 +8,6 @@ import ( "github.com/golang/protobuf/ptypes" "github.com/hashicorp/vault/helper/identity" "github.com/hashicorp/vault/helper/namespace" - "github.com/hashicorp/vault/helper/storagepacker" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/logical" ) @@ -396,15 +395,11 @@ func (i *IdentityStore) pathAliasIDDelete() framework.OperationFunc { return nil, err } - // Persist the entity object - entityAsAny, err := ptypes.MarshalAny(entity) + // Persist the entity + item, err := i.encodeEntity(entity) if err != nil { return nil, err } - item := &storagepacker.Item{ - ID: entity.ID, - Message: entityAsAny, - } err = i.entityPacker.PutItem(ctx, item) if err != nil { diff --git a/vault/identity_store_entities.go b/vault/identity_store_entities.go index aa02c1ddb6ed..1bc6e978dfe8 100644 --- a/vault/identity_store_entities.go +++ b/vault/identity_store_entities.go @@ -11,7 +11,6 @@ import ( memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/vault/helper/identity" "github.com/hashicorp/vault/helper/namespace" - "github.com/hashicorp/vault/helper/storagepacker" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/consts" "github.com/hashicorp/vault/sdk/helper/strutil" @@ -724,15 +723,10 @@ func (i *IdentityStore) mergeEntity(ctx context.Context, txn *memdb.Txn, toEntit if persist && !isPerfSecondaryOrStandby { // Persist the entity which we are merging to - toEntityAsAny, err := ptypes.MarshalAny(toEntity) + item, err := i.encodeEntity(toEntity) if err != nil { return nil, err } - item := &storagepacker.Item{ - ID: toEntity.ID, - Message: toEntityAsAny, - } - err = i.entityPacker.PutItem(ctx, item) if err != nil { return nil, err diff --git a/vault/identity_store_util.go b/vault/identity_store_util.go index afc2d28e199b..1da043a0f8f0 100644 --- a/vault/identity_store_util.go +++ b/vault/identity_store_util.go @@ -8,6 +8,7 @@ import ( "sync" "sync/atomic" + proto "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" "github.com/hashicorp/errwrap" memdb "github.com/hashicorp/go-memdb" @@ -36,7 +37,7 @@ func (c *Core) loadIdentityStoreArtifacts(ctx context.Context) error { } // Check for the legacy -> v2 upgrade case - upgradeLegacyStoragePacker := func(prefix string, packer storagepacker.StoragePacker) error { + upgradeLegacyStoragePacker := func(prefix string, packer storagepacker.StoragePacker, kind string) error { c.logger.Trace("checking for identity storagepacker upgrade", "prefix", prefix) bucketStorageView := logical.NewStorageView(c.identityStore.view, prefix+"buckets/") vals, err := bucketStorageView.List(ctx, "") @@ -75,6 +76,28 @@ func (c *Core) loadIdentityStoreArtifacts(ctx context.Context) error { } // Set to the new prefix for _, item := range bucket.Items { + // Parse the entity or group from the Message (proto.Any) field + // and re-encode it as a proto message into the 'Data' field. + switch kind { + case "entity": + entity, err := c.identityStore.decodeEntity(item) + if err != nil { + return err + } + item, err = c.identityStore.encodeEntity(entity) + if err != nil { + return err + } + case "group": + group, err := c.identityStore.decodeGroup(item) + if err != nil { + return err + } + item, err = c.identityStore.encodeGroup(group) + if err != nil { + return err + } + } packer.PutItem(ctx, item) } } @@ -103,10 +126,10 @@ func (c *Core) loadIdentityStoreArtifacts(ctx context.Context) error { return nil } - if err := upgradeLegacyStoragePacker(entityStoragePackerPrefix, c.identityStore.entityPacker); err != nil { + if err := upgradeLegacyStoragePacker(entityStoragePackerPrefix, c.identityStore.entityPacker, "entity"); err != nil { return err } - if err := upgradeLegacyStoragePacker(groupStoragePackerPrefix, c.identityStore.groupPacker); err != nil { + if err := upgradeLegacyStoragePacker(groupStoragePackerPrefix, c.identityStore.groupPacker, "group"); err != nil { return err } @@ -147,6 +170,74 @@ func (c *Core) loadIdentityStoreArtifacts(ctx context.Context) error { return loadFunc(ctx) } +func (i *IdentityStore) encodeEntity(entity *identity.Entity) (*storagepacker.Item, error) { + if entity == nil { + return nil, fmt.Errorf("nil entity") + } + + entityProto, err := proto.Marshal(entity) + if err != nil { + return nil, err + } + + return &storagepacker.Item{ + ID: entity.ID, + Data: entityProto, + }, nil +} + +func (i *IdentityStore) encodeGroup(group *identity.Group) (*storagepacker.Item, error) { + if group == nil { + return nil, fmt.Errorf("nil group") + } + + groupProto, err := proto.Marshal(group) + if err != nil { + return nil, err + } + + return &storagepacker.Item{ + ID: group.ID, + Data: groupProto, + }, nil +} + +func (i *IdentityStore) decodeGroup(item *storagepacker.Item) (*identity.Group, error) { + var group identity.Group + if item.Message != nil { + err := ptypes.UnmarshalAny(item.Message, &group) + if err != nil { + return nil, err + } + return &group, nil + } + + err := proto.Unmarshal(item.Data, &group) + if err != nil { + return nil, err + } + + return &group, nil +} + +func (i *IdentityStore) decodeEntity(item *storagepacker.Item) (*identity.Entity, error) { + var entity identity.Entity + if item.Message != nil { + err := ptypes.UnmarshalAny(item.Message, &entity) + if err != nil { + return nil, err + } + return &entity, nil + } + + err := proto.Unmarshal(item.Data, &entity) + if err != nil { + return nil, err + } + + return &entity, nil +} + func (i *IdentityStore) sanitizeName(name string) string { if i.disableLowerCasedNames { return name @@ -248,8 +339,8 @@ func (i *IdentityStore) loadGroups(ctx context.Context) error { // Need to check both map and Items in case it's during upgrading items := make([]*storagepacker.Item, 0, len(bucket.Items)+len(bucket.ItemMap)) items = append(items, bucket.Items...) - for id, message := range bucket.ItemMap { - items = append(items, &storagepacker.Item{ID: id, Message: message}) + for id, data := range bucket.ItemMap { + items = append(items, &storagepacker.Item{ID: id, Data: data}) } for _, item := range items { group, err := i.parseGroupFromBucketItem(item) @@ -424,8 +515,8 @@ func (i *IdentityStore) loadEntities(ctx context.Context) error { items := make([]*storagepacker.Item, 0, len(bucket.Items)+len(bucket.ItemMap)) items = append(items, bucket.Items...) - for id, message := range bucket.ItemMap { - items = append(items, &storagepacker.Item{ID: id, Message: message}) + for id, data := range bucket.ItemMap { + items = append(items, &storagepacker.Item{ID: id, Data: data}) } for _, item := range items { entity, err := i.parseEntityFromBucketItem(ctx, item) @@ -581,14 +672,12 @@ func (i *IdentityStore) upsertEntityInTxn(ctx context.Context, txn *memdb.Txn, e if persist { // Persist the previous entity object - marshaledPreviousEntity, err := ptypes.MarshalAny(previousEntity) + item, err := i.encodeEntity(previousEntity) if err != nil { return err } - err = i.entityPacker.PutItem(ctx, &storagepacker.Item{ - ID: previousEntity.ID, - Message: marshaledPreviousEntity, - }) + + err = i.entityPacker.PutItem(ctx, item) if err != nil { return err } @@ -602,16 +691,11 @@ func (i *IdentityStore) upsertEntityInTxn(ctx context.Context, txn *memdb.Txn, e } if persist { - entityAsAny, err := ptypes.MarshalAny(entity) + // Persist the entity + item, err := i.encodeEntity(entity) if err != nil { return err } - item := &storagepacker.Item{ - ID: entity.ID, - Message: entityAsAny, - } - - // Persist the entity object err = i.entityPacker.PutItem(ctx, item) if err != nil { return err @@ -1552,16 +1636,10 @@ func (i *IdentityStore) UpsertGroupInTxn(ctx context.Context, txn *memdb.Txn, gr } if persist { - groupAsAny, err := ptypes.MarshalAny(group) + item, err := i.encodeGroup(group) if err != nil { return err } - - item := &storagepacker.Item{ - ID: group.ID, - Message: groupAsAny, - } - sent, err := sendGroupUpgrade(i, group) if err != nil { return err