serializer.go

/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package lifecycle

import (
	"bytes"
	"fmt"
	"reflect"

	lb "github.com/hyperledger/fabric-protos-go/peer/lifecycle"
	"github.com/hyperledger/fabric/common/util"

	"github.com/golang/protobuf/proto"
	"github.com/pkg/errors"
)

const (
	MetadataInfix = "metadata"
	FieldsInfix   = "fields"
)

type ReadWritableState interface {
	ReadableState
	PutState(key string, value []byte) error
	DelState(key string) error
}

type ReadableState interface {
	GetState(key string) (value []byte, err error)
}

type OpaqueState interface {
	GetStateHash(key string) (value []byte, err error)
	CollectionName() string
}

type RangeableState interface {
	GetStateRange(prefix string) (map[string][]byte, error)
}

type Marshaler func(proto.Message) ([]byte, error)

func (m Marshaler) Marshal(msg proto.Message) ([]byte, error) {
	if m != nil {
		return m(msg)
	}
	return proto.Marshal(msg)
}

var ProtoMessageType = reflect.TypeOf((*proto.Message)(nil)).Elem()

// Serializer is used to write structures into the db and to read them back out.
// Although it's unfortunate to write a custom serializer, rather than to use something
// pre-written, like protobuf or JSON, in order to produce precise readwrite sets which
// only perform state updates for keys which are actually updated (and not simply set
// to the same value again) custom serialization is required.
type Serializer struct {
	// Marshaler, when nil uses the standard protobuf impl.
	// Can be overridden for test.
	Marshaler Marshaler
}

// SerializableChecks performs some boilerplate checks to make sure the given structure
// is serializable.  It returns the reflected version of the value and a slice of all
// field names, or an error.
func (s *Serializer) SerializableChecks(structure interface{}) (reflect.Value, []string, error) {
	value := reflect.ValueOf(structure)
	if value.Kind() != reflect.Ptr {
		return reflect.Value{}, nil, errors.Errorf("must be pointer to struct, but got non-pointer %v", value.Kind())
	}

	value = value.Elem()
	if value.Kind() != reflect.Struct {
		return reflect.Value{}, nil, errors.Errorf("must be pointers to struct, but got pointer to %v", value.Kind())
	}

	allFields := make([]string, value.NumField())
	for i := 0; i < value.NumField(); i++ {
		fieldName := value.Type().Field(i).Name
		fieldValue := value.Field(i)
		allFields[i] = fieldName
		switch fieldValue.Kind() {
		case reflect.String:
		case reflect.Int64:
		case reflect.Slice:
			if fieldValue.Type().Elem().Kind() != reflect.Uint8 {
				return reflect.Value{}, nil, errors.Errorf("unsupported slice type %v for field %s", fieldValue.Type().Elem().Kind(), fieldName)
			}
		case reflect.Ptr:
			if !fieldValue.Type().Implements(ProtoMessageType) {
				return reflect.Value{}, nil, errors.Errorf("unsupported pointer type %v for field %s (must be proto)", fieldValue.Type().Elem(), fieldName)
			}
		default:
			return reflect.Value{}, nil, errors.Errorf("unsupported structure field kind %v for serialization for field %s", fieldValue.Kind(), fieldName)
		}
	}
	return value, allFields, nil
}

// Serialize takes a pointer to a struct, and writes each of its fields as keys
// into a namespace.  It also writes the struct metadata (if it needs updating)
// and,  deletes any keys in the namespace which are not found in the struct.
// Note: If a key already exists for the field, and the value is unchanged, then
// the key is _not_ written to.
func (s *Serializer) Serialize(namespace, name string, structure interface{}, state ReadWritableState) error {
	value, allFields, err := s.SerializableChecks(structure)
	if err != nil {
		return errors.WithMessagef(err, "structure for namespace %s/%s is not serializable", namespace, name)
	}

	metadata, ok, err := s.DeserializeMetadata(namespace, name, state)
	if err != nil {
		return errors.WithMessagef(err, "could not deserialize metadata for namespace %s/%s", namespace, name)
	}
	if !ok {
		metadata = &lb.StateMetadata{}
	}

	existingKeys := map[string][]byte{}
	for _, existingField := range metadata.Fields {
		fqKey := FieldKey(namespace, name, existingField)
		value, err := state.GetState(fqKey)
		if err != nil {
			return errors.WithMessagef(err, "could not get value for key %s", fqKey)
		}
		existingKeys[fqKey] = value
	}

	for i := 0; i < value.NumField(); i++ {
		fieldName := value.Type().Field(i).Name
		fieldValue := value.Field(i)

		keyName := FieldKey(namespace, name, fieldName)

		stateData := &lb.StateData{}
		switch fieldValue.Kind() {
		case reflect.String:
			stateData.Type = &lb.StateData_String_{String_: fieldValue.String()}
		case reflect.Int64:
			stateData.Type = &lb.StateData_Int64{Int64: fieldValue.Int()}
		case reflect.Slice:
			stateData.Type = &lb.StateData_Bytes{Bytes: fieldValue.Bytes()}
		case reflect.Ptr:
			var bin []byte
			if !fieldValue.IsNil() {
				bin, err = s.Marshaler.Marshal(fieldValue.Interface().(proto.Message))
				if err != nil {
					return errors.Wrapf(err, "could not marshal field %s", fieldName)
				}
			}
			stateData.Type = &lb.StateData_Bytes{Bytes: bin}
			// Note, other field kinds and bad types have already been checked by SerializableChecks
		}

		marshaledFieldValue, err := s.Marshaler.Marshal(stateData)
		if err != nil {
			return errors.WithMessagef(err, "could not marshal value for key %s", keyName)
		}

		if existingValue, ok := existingKeys[keyName]; !ok || !bytes.Equal(existingValue, marshaledFieldValue) {
			err := state.PutState(keyName, marshaledFieldValue)
			if err != nil {
				return errors.WithMessage(err, "could not write key into state")
			}
		}
		delete(existingKeys, keyName)
	}

	typeName := value.Type().Name()
	if len(existingKeys) > 0 || typeName != metadata.Datatype || len(metadata.Fields) != value.NumField() {
		metadata.Datatype = typeName
		metadata.Fields = allFields
		newMetadataBin, err := s.Marshaler.Marshal(metadata)
		if err != nil {
			return errors.WithMessagef(err, "could not marshal metadata for namespace %s/%s", namespace, name)
		}
		err = state.PutState(MetadataKey(namespace, name), newMetadataBin)
		if err != nil {
			return errors.WithMessagef(err, "could not store metadata for namespace %s/%s", namespace, name)
		}
	}

	for key := range existingKeys {
		err := state.DelState(key)
		if err != nil {
			return errors.WithMessagef(err, "could not delete unneeded key %s", key)
		}
	}

	return nil
}

func (s *Serializer) IsMetadataSerialized(namespace, name string, structure interface{}, state OpaqueState) (bool, error) {
	value, allFields, err := s.SerializableChecks(structure)
	if err != nil {
		return false, errors.WithMessagef(err, "structure for namespace %s/%s is not serializable", namespace, name)
	}

	mdKey := MetadataKey(namespace, name)
	metadata := &lb.StateMetadata{
		Datatype: value.Type().Name(),
		Fields:   allFields,
	}

	metadataBin, err := s.Marshaler.Marshal(metadata)
	if err != nil {
		return false, errors.WithMessagef(err, "could not marshal metadata for namespace %s/%s", namespace, name)
	}

	existingMDHash, err := state.GetStateHash(mdKey)
	if err != nil {
		return false, errors.WithMessagef(err, "could not get state hash for metadata key %s", mdKey)
	}

	return bytes.Equal(util.ComputeSHA256(metadataBin), existingMDHash), nil
}

// IsSerialized essentially checks if the hashes of a serialized version of a structure matches the hashes
// of the pre-image of some struct serialized into the database.
func (s *Serializer) IsSerialized(namespace, name string, structure interface{}, state OpaqueState) (bool, error) {
	value, allFields, err := s.SerializableChecks(structure)
	if err != nil {
		return false, errors.WithMessagef(err, "structure for namespace %s/%s is not serializable", namespace, name)
	}

	fqKeys := make([]string, 0, len(allFields)+1)
	fqKeys = append(fqKeys, MetadataKey(namespace, name))
	for _, field := range allFields {
		fqKeys = append(fqKeys, FieldKey(namespace, name, field))
	}

	existingKeys := map[string][]byte{}
	for _, fqKey := range fqKeys {
		value, err := state.GetStateHash(fqKey)
		if err != nil {
			return false, errors.WithMessagef(err, "could not get value for key %s", fqKey)
		}
		existingKeys[fqKey] = value
	}

	metadata := &lb.StateMetadata{
		Datatype: value.Type().Name(),
		Fields:   allFields,
	}
	metadataBin, err := s.Marshaler.Marshal(metadata)
	if err != nil {
		return false, errors.WithMessagef(err, "could not marshal metadata for namespace %s/%s", namespace, name)
	}

	metadataKeyName := MetadataKey(namespace, name)
	if !bytes.Equal(util.ComputeSHA256(metadataBin), existingKeys[metadataKeyName]) {
		return false, nil
	}

	for i := 0; i < value.NumField(); i++ {
		fieldName := value.Type().Field(i).Name
		fieldValue := value.Field(i)

		keyName := FieldKey(namespace, name, fieldName)

		stateData := &lb.StateData{}
		switch fieldValue.Kind() {
		case reflect.String:
			stateData.Type = &lb.StateData_String_{String_: fieldValue.String()}
		case reflect.Int64:
			stateData.Type = &lb.StateData_Int64{Int64: fieldValue.Int()}
		case reflect.Slice:
			stateData.Type = &lb.StateData_Bytes{Bytes: fieldValue.Bytes()}
		case reflect.Ptr:
			var bin []byte
			if !fieldValue.IsNil() {
				bin, err = s.Marshaler.Marshal(fieldValue.Interface().(proto.Message))
				if err != nil {
					return false, errors.Wrapf(err, "could not marshal field %s", fieldName)
				}
			}
			stateData.Type = &lb.StateData_Bytes{Bytes: bin}
			// Note, other field kinds and bad types have already been checked by SerializableChecks
		}

		marshaledFieldValue, err := s.Marshaler.Marshal(stateData)
		if err != nil {
			return false, errors.WithMessagef(err, "could not marshal value for key %s", keyName)
		}

		if existingValue, ok := existingKeys[keyName]; !ok || !bytes.Equal(existingValue, util.ComputeSHA256(marshaledFieldValue)) {
			return false, nil
		}
	}

	return true, nil
}

// Deserialize accepts a struct (of a type previously serialized) and populates it with the values from the db.
// Note: The struct names for the serialization and deserialization must match exactly.  Unencoded fields are not
// populated, and the extraneous keys are ignored.  The metadata provided should have been returned by a DeserializeMetadata
// call for the same namespace and name.
func (s *Serializer) Deserialize(namespace, name string, metadata *lb.StateMetadata, structure interface{}, state ReadableState) error {
	value, _, err := s.SerializableChecks(structure)
	if err != nil {
		return errors.WithMessagef(err, "could not deserialize namespace %s/%s to unserializable type %T", namespace, name, structure)
	}

	typeName := value.Type().Name()
	if typeName != metadata.Datatype {
		return errors.Errorf("type name mismatch '%s' != '%s'", typeName, metadata.Datatype)
	}

	for i := 0; i < value.NumField(); i++ {
		fieldName := value.Type().Field(i).Name
		fieldValue := value.Field(i)
		switch fieldValue.Kind() {
		case reflect.String:
			oneOf, err := s.DeserializeFieldAsString(namespace, name, fieldName, state)
			if err != nil {
				return err
			}
			fieldValue.SetString(oneOf)
		case reflect.Int64:
			oneOf, err := s.DeserializeFieldAsInt64(namespace, name, fieldName, state)
			if err != nil {
				return err
			}
			fieldValue.SetInt(oneOf)
		case reflect.Slice:
			oneOf, err := s.DeserializeFieldAsBytes(namespace, name, fieldName, state)
			if err != nil {
				return err
			}
			if oneOf != nil {
				fieldValue.SetBytes(oneOf)
			}
		case reflect.Ptr:
			// Note, even non-existant keys will decode to an empty proto
			msg := reflect.New(fieldValue.Type().Elem())
			err := s.DeserializeFieldAsProto(namespace, name, fieldName, state, msg.Interface().(proto.Message))
			if err != nil {
				return err
			}
			fieldValue.Set(msg)
			// Note, other field kinds and bad types have already been checked by SerializableChecks
		}
	}

	return nil
}

func MetadataKey(namespace, name string) string {
	return fmt.Sprintf("%s/%s/%s", namespace, MetadataInfix, name)
}

func FieldKey(namespace, name, field string) string {
	return fmt.Sprintf("%s/%s/%s/%s", namespace, FieldsInfix, name, field)
}

func (s *Serializer) DeserializeMetadata(namespace, name string, state ReadableState) (*lb.StateMetadata, bool, error) {
	metadataBin, err := state.GetState(MetadataKey(namespace, name))
	if err != nil {
		return nil, false, errors.WithMessagef(err, "could not query metadata for namespace %s/%s", namespace, name)
	}
	if metadataBin == nil {
		return nil, false, nil
	}

	metadata := &lb.StateMetadata{}
	err = proto.Unmarshal(metadataBin, metadata)
	if err != nil {
		return nil, false, errors.Wrapf(err, "could not unmarshal metadata for namespace %s/%s", namespace, name)
	}

	return metadata, true, nil
}

func (s *Serializer) DeserializeField(namespace, name, field string, state ReadableState) (*lb.StateData, error) {
	keyName := FieldKey(namespace, name, field)
	value, err := state.GetState(keyName)
	if err != nil {
		return nil, errors.WithMessagef(err, "could not get state for key %s", keyName)
	}

	stateData := &lb.StateData{}
	err = proto.Unmarshal(value, stateData)
	if err != nil {
		return nil, errors.Wrapf(err, "could not unmarshal state for key %s", keyName)
	}

	return stateData, nil
}

func (s *Serializer) DeserializeFieldAsString(namespace, name, field string, state ReadableState) (string, error) {
	value, err := s.DeserializeField(namespace, name, field, state)
	if err != nil {
		return "", err
	}
	if value.Type == nil {
		return "", nil
	}
	oneOf, ok := value.Type.(*lb.StateData_String_)
	if !ok {
		return "", errors.Errorf("expected key %s/fields/%s/%s to encode a value of type String, but was %T", namespace, name, field, value.Type)
	}
	return oneOf.String_, nil
}

func (s *Serializer) DeserializeFieldAsBytes(namespace, name, field string, state ReadableState) ([]byte, error) {
	value, err := s.DeserializeField(namespace, name, field, state)
	if err != nil {
		return nil, err
	}
	if value.Type == nil {
		return nil, nil
	}
	oneOf, ok := value.Type.(*lb.StateData_Bytes)
	if !ok {
		return nil, errors.Errorf("expected key %s to encode a value of type []byte, but was %T", FieldKey(namespace, name, field), value.Type)
	}
	return oneOf.Bytes, nil
}

func (s *Serializer) DeserializeFieldAsProto(namespace, name, field string, state ReadableState, msg proto.Message) error {
	bin, err := s.DeserializeFieldAsBytes(namespace, name, field, state)
	if err != nil {
		return err
	}
	err = proto.Unmarshal(bin, msg)
	if err != nil {
		return errors.Wrapf(err, "could not unmarshal key %s to %T", FieldKey(namespace, name, field), msg)
	}
	return nil
}

func (s *Serializer) DeserializeFieldAsInt64(namespace, name, field string, state ReadableState) (int64, error) {
	value, err := s.DeserializeField(namespace, name, field, state)
	if err != nil {
		return 0, err
	}
	if value.Type == nil {
		return 0, nil
	}
	oneOf, ok := value.Type.(*lb.StateData_Int64)
	if !ok {
		return 0, errors.Errorf("expected key %s to encode a value of type Int64, but was %T", FieldKey(namespace, name, field), value.Type)
	}
	return oneOf.Int64, nil
}

func (s *Serializer) DeserializeAllMetadata(namespace string, state RangeableState) (map[string]*lb.StateMetadata, error) {
	prefix := fmt.Sprintf("%s/%s/", namespace, MetadataInfix)
	kvs, err := state.GetStateRange(prefix)
	if err != nil {
		return nil, errors.WithMessagef(err, "could not get state range for namespace %s", namespace)
	}
	result := map[string]*lb.StateMetadata{}
	for key, value := range kvs {
		name := key[len(prefix):]
		metadata := &lb.StateMetadata{}
		err = proto.Unmarshal(value, metadata)
		if err != nil {
			return nil, errors.Wrapf(err, "error unmarshaling metadata for key %s", key)
		}
		result[name] = metadata
	}
	return result, nil
}