StoragePackerV2: Item storage from proto.Any to []byte

helper/storagepacker/storagepacker_v2.go

@@ -12,7 +12,6 @@ import (
 
 	radix "github.com/armon/go-radix"
 	"github.com/golang/protobuf/proto"
-	any "github.com/golang/protobuf/ptypes/any"
 	"github.com/hashicorp/errwrap"
 	log "github.com/hashicorp/go-hclog"
 	multierror "github.com/hashicorp/go-multierror"
@@ -312,7 +311,7 @@ func (s *StoragePackerV2) shardBucket(ctx context.Context, bucket *LockedBucket,
 			LockEntry: lock,
 			Bucket: &Bucket{
 				Key:     fmt.Sprintf("%s/%s", bucket.Key, shardKey),
-				ItemMap: make(map[string]*any.Any),
+				ItemMap: make(map[string][]byte),
 			},
 		}
 		shards[shardKey] = shardedBucket
@@ -472,12 +471,12 @@ func (s *LockedBucket) upsert(item *Item) error {
 	}
 
 	if s.ItemMap == nil {
-		s.ItemMap = make(map[string]*any.Any)
+		s.ItemMap = make(map[string][]byte)
 	}
 
 	itemHash := GetItemIDHash(item.ID)
 
-	s.ItemMap[itemHash] = item.Message
+	s.ItemMap[itemHash] = item.Data
 	return nil
 }
 
@@ -591,14 +590,14 @@ func (s *StoragePackerV2) GetItem(ctx context.Context, itemID string) (*Item, er
 
 	itemHash := GetItemIDHash(itemID)
 
-	item, ok := bucket.ItemMap[itemHash]
+	data, ok := bucket.ItemMap[itemHash]
 	if !ok {
 		return nil, nil
 	}
 
 	return &Item{
-		ID:      itemID,
-		Message: item,
+		ID:   itemID,
+		Data: data,
 	}, nil
 }
 
@@ -612,6 +611,14 @@ func (s *StoragePackerV2) PutItem(ctx context.Context, item *Item) error {
 		return fmt.Errorf("missing ID in item")
 	}
 
+	if item.Data == nil {
+		return fmt.Errorf("missing data in item")
+	}
+
+	if item.Message != nil {
+		return fmt.Errorf("'Message' is deprecated; use 'Data' instead")
+	}
+
 	// Get the bucket key
 	bucketKey := s.BucketStorageKeyForItemID(item.ID)
 	cacheKey := s.GetCacheKey(bucketKey)

helper/storagepacker/types.proto

@@ -14,8 +14,11 @@ message Item {
 	// described above, the caller *must not* rely on this value to be
 	// consistent with what they passed in.
 	string id = 1;
-	// message is the contents of the item
+	// Message holds the contents of the item
+	// Deprecated: Use 'Data' instead
 	google.protobuf.Any message = 2;
+	// Data holds the contents of the item. Used in storage packer v2.
+	bytes data = 3;
 }
 
 // Bucket is a construct to hold multiple items within itself. This
@@ -30,5 +33,5 @@ message Bucket {
 	// Items holds the items contained within this bucket. Used by v1.
 	repeated Item items = 2;
 	// ItemMap stores a mapping of item ID to message. Used by v2.
-	map <string, google.protobuf.Any> item_map = 3;
+	map <string, bytes> item_map = 3;
 }

vault/external_tests/storagepacker/storagepacker_sharding_test.go

@@ -9,13 +9,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/golang/protobuf/ptypes"
 	consulapi "github.com/hashicorp/consul/api"
 	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/helper/storagepacker"
 	"github.com/hashicorp/vault/helper/testhelpers/consul"
 	vaulthttp "github.com/hashicorp/vault/http"
 	physConsul "github.com/hashicorp/vault/physical/consul"
+	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/plugin/pb"
@@ -93,18 +93,18 @@ func TestStoragePacker_Sharding(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	protoSecret := &pb.Secret{
+	secret := &pb.Secret{
 		InternalData: randString,
 	}
-	messageAsAny, err := ptypes.MarshalAny(protoSecret)
+	secretJSON, err := jsonutil.EncodeJSON(secret)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	for i := 0; i < numEntries; i++ {
 		if err := packer.PutItem(ctx, &storagepacker.Item{
-			ID:      fmt.Sprintf("%05d", i),
-			Message: messageAsAny,
+			ID:   fmt.Sprintf("%05d", i),
+			Data: secretJSON,
 		}); err != nil {
 			t.Fatal(err)
 		}

vault/identity_store.go

@@ -177,7 +177,7 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) {
 			items := make([]*storagepacker.Item, 0, len(bucket.Items)+len(bucket.ItemMap))
 			items = append(items, bucket.Items...)
 			for id, message := range bucket.ItemMap {
-				items = append(items, &storagepacker.Item{ID: id, Message: message})
+				items = append(items, &storagepacker.Item{ID: id, Data: message})
 			}
 			for _, item := range items {
 				entity, err := i.parseEntityFromBucketItem(ctx, item)
@@ -246,7 +246,7 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) {
 			items := make([]*storagepacker.Item, 0, len(bucket.Items)+len(bucket.ItemMap))
 			items = append(items, bucket.Items...)
 			for id, message := range bucket.ItemMap {
-				items = append(items, &storagepacker.Item{ID: id, Message: message})
+				items = append(items, &storagepacker.Item{ID: id, Data: message})
 			}
 			for _, item := range items {
 				group, err := i.parseGroupFromBucketItem(item)
@@ -302,8 +302,7 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto
 
 	persistNeeded := false
 
-	var entity identity.Entity
-	err := ptypes.UnmarshalAny(item.Message, &entity)
+	entity, err := i.decodeEntity(item)
 	if err != nil {
 		// If we encounter an error, it would mean that the format of the
 		// entity is an older one. Try decoding using the older format and if
@@ -349,7 +348,7 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto
 
 	entity.BucketKey = i.entityPacker.BucketKey(entity.ID)
 
-	pN, err := parseExtraEntityFromBucket(ctx, i, &entity)
+	pN, err := parseExtraEntityFromBucket(ctx, i, entity)
 	if err != nil {
 		return nil, err
 	}
@@ -358,16 +357,11 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto
 	}
 
 	if persistNeeded && !i.core.ReplicationState().HasState(consts.ReplicationPerformanceSecondary) {
-		entityAsAny, err := ptypes.MarshalAny(&entity)
+		item, err := i.encodeEntity(entity)
 		if err != nil {
 			return nil, err
 		}
 
-		item := &storagepacker.Item{
-			ID:      entity.ID,
-			Message: entityAsAny,
-		}
-
 		// Store the entity with new format
 		err = i.entityPacker.PutItem(ctx, item)
 		if err != nil {
@@ -379,16 +373,15 @@ func (i *IdentityStore) parseEntityFromBucketItem(ctx context.Context, item *sto
 		entity.NamespaceID = namespace.RootNamespaceID
 	}
 
-	return &entity, nil
+	return entity, nil
 }
 
 func (i *IdentityStore) parseGroupFromBucketItem(item *storagepacker.Item) (*identity.Group, error) {
 	if item == nil {
 		return nil, fmt.Errorf("nil item")
 	}
 
-	var group identity.Group
-	err := ptypes.UnmarshalAny(item.Message, &group)
+	group, err := i.decodeGroup(item)
 	if err != nil {
 		return nil, errwrap.Wrapf("failed to decode group from storage bucket item: {{err}}", err)
 	}
@@ -399,7 +392,7 @@ func (i *IdentityStore) parseGroupFromBucketItem(item *storagepacker.Item) (*ide
 
 	group.BucketKey = i.groupPacker.BucketKey(group.ID)
 
-	return &group, nil
+	return group, nil
 }
 
 // entityByAliasFactors fetches the entity based on factors of alias, i.e mount

vault/identity_store_aliases.go

@@ -8,7 +8,6 @@ import (
 	"github.com/golang/protobuf/ptypes"
 	"github.com/hashicorp/vault/helper/identity"
 	"github.com/hashicorp/vault/helper/namespace"
-	"github.com/hashicorp/vault/helper/storagepacker"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/logical"
 )
@@ -396,15 +395,11 @@ func (i *IdentityStore) pathAliasIDDelete() framework.OperationFunc {
 			return nil, err
 		}
 
-		// Persist the entity object
-		entityAsAny, err := ptypes.MarshalAny(entity)
+		// Persist the entity
+		item, err := i.encodeEntity(entity)
 		if err != nil {
 			return nil, err
 		}
-		item := &storagepacker.Item{
-			ID:      entity.ID,
-			Message: entityAsAny,
-		}
 
 		err = i.entityPacker.PutItem(ctx, item)
 		if err != nil {

vault/identity_store_entities.go

@@ -11,7 +11,6 @@ import (
 	memdb "github.com/hashicorp/go-memdb"
 	"github.com/hashicorp/vault/helper/identity"
 	"github.com/hashicorp/vault/helper/namespace"
-	"github.com/hashicorp/vault/helper/storagepacker"
 	"github.com/hashicorp/vault/sdk/framework"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/strutil"
@@ -724,15 +723,10 @@ func (i *IdentityStore) mergeEntity(ctx context.Context, txn *memdb.Txn, toEntit
 
 	if persist && !isPerfSecondaryOrStandby {
 		// Persist the entity which we are merging to
-		toEntityAsAny, err := ptypes.MarshalAny(toEntity)
+		item, err := i.encodeEntity(toEntity)
 		if err != nil {
 			return nil, err
 		}
-		item := &storagepacker.Item{
-			ID:      toEntity.ID,
-			Message: toEntityAsAny,
-		}
-
 		err = i.entityPacker.PutItem(ctx, item)
 		if err != nil {
 			return nil, err