From d235ebdb175d2782cce57b0c93ad02b24e41a4c0 Mon Sep 17 00:00:00 2001 From: Michael Lam Date: Sat, 3 Aug 2019 10:56:38 -0700 Subject: [PATCH 1/6] 1) Add comments 2) minor refactors 3) fix panic by RandInt64Between if spread is larger than Int64Max 4) Fix problems with RandIntBetween() and RandInt() on 32 bit systems 5) Fix tests which had problems due to assumption the 'between' functions produce [min,max] when in fact they are [min,max) 6) Add test to show RandInt64Between() fails if spread of min/max was too large --- common/primitives/random/random.go | 103 +++++++++++++++++++----- common/primitives/random/random_test.go | 55 +++++++++++-- common/primitives/varint.go | 11 ++- 3 files changed, 140 insertions(+), 29 deletions(-) diff --git a/common/primitives/random/random.go b/common/primitives/random/random.go index fe1c40656a..b31f6dda20 100644 --- a/common/primitives/random/random.go +++ b/common/primitives/random/random.go @@ -8,24 +8,42 @@ import ( "crypto/rand" "math" "math/big" + "strconv" ) +// UintSize is either 32 or 64. Bitwise complement 0 to all 1's, bitshift 32. +// a) Will be 0 on 32 bit systems causing 0 & 1 to be zero, leaving 32. +// b) Will be 1 on 64 bit systems causing 1 & 1 to be 1, shifting 32 to 64 +const UintSize = 32 << (^uint(0) >> 32 & 1) // 32 or 64 + +// Define maximum int sizes (correct on 32 bit or 64 bit systems) +const ( + MaxInt = 1<<(UintSize-1) - 1 // 1<<31 - 1 or 1<<63 - 1 + MinInt = -MaxInt - 1 // -1 << 31 or -1 << 63 + MaxUint = 1<=max then returns 0. func RandUInt64Between(min, max uint64) uint64 { if min >= max { return 0 @@ -39,51 +57,92 @@ func RandUInt64Between(min, max uint64) uint64 { return randnum.Uint64() } +// RandInt64 returns a random number between [0,MaxInt64). Note - even though the return int64 is signed, +// the underlying function call should never return a negative value. func RandInt64() int64 { int64max := big.NewInt(math.MaxInt64) randnum, _ := rand.Int(rand.Reader, int64max) return randnum.Int64() } +// RandInt64Between returns a random number between [min,max). If min>=max then returns 0. It takes special care +// to ensure the int64 difference between max-min doesn't overflow. Worst case int64 can support from -N to N-1 +// which if input into this system as max/min creates (N-1)-(-N) = 2*N-1 number which would overflow int64. To +// deal with this, we make max-min a uint64 and get a random uint64. If the random number is larger than int64 +// maximum value, then we first add the max int64 to the min value, and subtract the max int64 from the random +// uint64 number, thus breaking the addition into two smaller pieces which each fit within the int64 size func RandInt64Between(min, max int64) int64 { if min >= max { return 0 } - int64max := big.NewInt(max - min) - randnum, _ := rand.Int(rand.Reader, int64max) - m := big.NewInt(min) - randnum = randnum.Add(randnum, m) + + // A Uint64 allows the full width of max-min to be stored + therange := big.NewInt(0) + therange.SetUint64(uint64(max - min)) + randnum, _ := rand.Int(rand.Reader, therange) // randnum is potentially larger than int64 can handle + m := big.NewInt(0) + m.SetInt64(min) + if randnum.Uint64() > math.MaxInt64 { + // The random number is larger than the maximum int64 and must use uint64 math + // Note: The only way this is possible is if the min is negative (necessary but not sufficient to be here) + maxInt64 := big.NewInt(math.MaxInt64) + m = m.Add(maxInt64, m) // First add the MaxInt64 number to the input min + randnum = randnum.Sub(randnum, maxInt64) // Then make randnum the residual (subtract MaxInt64) + } + randnum = randnum.Add(randnum, m) // We either directly add the randnum (or the residual from large number) + return randnum.Int64() } -func RandInt() int { - return int(RandInt64()) +// RandIntBetween returns a random number between [min,max). If min>=max then returns 0. The same special care +// is used as explained in the above RandInt64Between function. Here we never allow something larger than +// MaxInt to be added to the int, which preserves safe addition if we are on a 32 bit or 64 bit system. +func RandIntBetween(min, max int) int { + if min >= max { + return 0 + } + + // A Uint64 allows the full width of max-min to be stored + therange := big.NewInt(0) + therange.SetUint64(uint64(max - min)) + randnum, _ := rand.Int(rand.Reader, therange) // randnum is potentially larger than int can handle + m := big.NewInt(0) + m.SetInt64(int64(min)) + if randnum.Uint64() > MaxInt { + // The random number is larger than the maximum int and so we use careful math to avoid integer oveflow + maxInt := big.NewInt(int64(MaxInt)) + m = m.Add(maxInt, m) // First add the MaxInt number to the input min + randnum = randnum.Sub(randnum, maxInt) // Then make randnum the residual (subtract MaxInt) + } + randnum = randnum.Add(randnum, m) // We either directly add the randnum (or the residual from large number) + + // To go back from a big.Int to a standard int, we write to a string and then convert it back to an int + randstringnum := randnum.String() + randnumint, err := strconv.Atoi(randstringnum) + if err != nil { + panic("RandIntBetween: Unable to convert string to int") + } + return randnumint } -func RandIntBetween(min, max int) int { - return int(RandInt64Between(int64(min), int64(max))) +// RandInt returns a random integer between [0,MaxInt64) +func RandInt() int { + return RandIntBetween(0, MaxInt) } +// RandByteSlice returns a random byte slice of length 0 <= len <= 63 func RandByteSlice() []byte { l := RandInt() % 64 - answer := make([]byte, l) - _, err := rand.Read(answer) - if err != nil { - return nil - } - return answer + return RandByteSliceOfLen(l) } +// RandNonEmptyByteSlice returns a random byte slice of length 1 <= len <= 63 (guaranteed not zero length) func RandNonEmptyByteSlice() []byte { l := RandInt()%63 + 1 - answer := make([]byte, l) - _, err := rand.Read(answer) - if err != nil { - return nil - } - return answer + return RandByteSliceOfLen(l) } +// RandByteSliceOfLen returns a random byte slice of specified length func RandByteSliceOfLen(l int) []byte { if l <= 0 { return nil @@ -96,8 +155,12 @@ func RandByteSliceOfLen(l int) []byte { return answer } +// StringAlphabet is the valid characters supported by RandomString() below const StringAlphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()" +// RandomString creates a random string of length 0 <= len < 128 consisting only of characters within +// the set of lower and uppercase letters, numbers 0-9, and the special characters associated with 'shift+' +// on your keyboard func RandomString() string { l := RandIntBetween(0, 128) answer := []byte{} diff --git a/common/primitives/random/random_test.go b/common/primitives/random/random_test.go index 4c50a14b9a..a50e976754 100644 --- a/common/primitives/random/random_test.go +++ b/common/primitives/random/random_test.go @@ -5,11 +5,13 @@ package random_test import ( + "math" "testing" . "github.com/FactomProject/factomd/common/primitives/random" ) +// TestRandUInt64Between checks that over N calls to the UInt64 random number generator, they all fall between [min,max) func TestRandUInt64Between(t *testing.T) { var min uint64 = 100 var max uint64 = 200 @@ -19,7 +21,7 @@ func TestRandUInt64Between(t *testing.T) { if r < min { t.Errorf("Returned value smaller than min - %v < %v", r, min) } - if r > max { + if r >= max { t.Errorf("Returned value greater than max - %v > %v", r, max) } @@ -29,6 +31,7 @@ func TestRandUInt64Between(t *testing.T) { } } +// TestRandInt64Between checks that over N calls to the Int64 random number generator, they all fall between [min,max) func TestRandInt64Between(t *testing.T) { var min int64 = -100 var max int64 = 200 @@ -38,7 +41,7 @@ func TestRandInt64Between(t *testing.T) { if r < min { t.Errorf("Returned value smaller than min - %v < %v", r, min) } - if r > max { + if r >= max { t.Errorf("Returned value greater than max - %v > %v", r, max) } @@ -48,6 +51,24 @@ func TestRandInt64Between(t *testing.T) { } } +// TestRandInt64BetweenBig checks that over N calls to the Int64 random number generator, they all fall between [min,max) +// It further uses a BIG range to verify no overflow happens because max-min can be ~ 2*|MaxInt64Value| +func TestRandInt64BetweenBig(t *testing.T) { + var max int64 = math.MaxInt64 + var min int64 = -max - 1 + + for i := 0; i < 1000; i++ { + r := RandInt64Between(min, max) + if r < min { + t.Errorf("Returned value smaller than min - %v < %v", r, min) + } + if r >= max { + t.Errorf("Returned value greater than max - %v > %v", r, max) + } + } +} + +// TestRandIntBetween checks that over N calls to the Int random number generator, they all fall between [min,max) func TestRandIntBetween(t *testing.T) { var min int = -100 var max int = 200 @@ -57,7 +78,7 @@ func TestRandIntBetween(t *testing.T) { if r < min { t.Errorf("Returned value smaller than min - %v < %v", r, min) } - if r > max { + if r >= max { t.Errorf("Returned value greater than max - %v > %v", r, max) } @@ -67,26 +88,46 @@ func TestRandIntBetween(t *testing.T) { } } +// TestRandIntBetweenBig checks that over N calls to the int random number generator, they all fall between [min,max) +// It further uses a BIG range to verify no overflow happens because max-min can be ~ 2*|MaxIntValue| +func TestRandIntBetweenBig(t *testing.T) { + var max int = MaxInt + var min int = MinInt + + for i := 0; i < 1000; i++ { + r := RandIntBetween(min, max) + if r < min { + t.Errorf("Returned value smaller than min - %v < %v", r, min) + } + if r >= max { + t.Errorf("Returned value greater than max - %v > %v", r, max) + } + } +} + +// TestRandByteSlice checks that over N calls to the random byte slice generator, none have lengths >= 64 func TestRandByteSlice(t *testing.T) { for i := 0; i < 1000; i++ { r := RandByteSlice() - if len(r) > 64 { + if len(r) >= 64 { t.Errorf("Returned a wrong size") } } } +// TestRandNonEmptyByteSlice checks that over N calls to the random byte slice generator, none have lengths >= 64, or ==0 func TestRandNonEmptyByteSlice(t *testing.T) { for i := 0; i < 1000; i++ { r := RandNonEmptyByteSlice() - if len(r) > 64 || len(r) == 0 { + if len(r) >= 64 || len(r) == 0 { t.Errorf("Returned a wrong size %v", len(r)) } } } +// TestRandByteSliceOfLen checks that N calls to the RandByteSliceOfLen(L) function always produces a slice of length L func TestRandByteSliceOfLen(t *testing.T) { for i := 0; i < 1000; i++ { r := RandByteSliceOfLen(100) @@ -97,16 +138,18 @@ func TestRandByteSliceOfLen(t *testing.T) { } } +// TestRandomString checks that N calls of RandomString() always produce a string of length <=128 func TestRandomString(t *testing.T) { for i := 0; i < 100; i++ { r := RandomString() - if len(r) > 128 { + if len(r) >= 128 { t.Errorf("Returned a wrong size %v", len(r)) } } } +// TestUintNumbers checks that N calls of RandUnitXX() generators never returns a negative value func TestUintNumbers(t *testing.T) { for i := 0; i < 1000; i++ { r := RandUInt64() diff --git a/common/primitives/varint.go b/common/primitives/varint.go index c8c728ce96..643648bb97 100644 --- a/common/primitives/varint.go +++ b/common/primitives/varint.go @@ -10,8 +10,9 @@ import ( "github.com/FactomProject/factomd/common/primitives/random" ) +// RandomVarInt returns a random variable integer func RandomVarInt() uint64 { - length := random.RandIntBetween(1, 10) + length := random.RandIntBetween(1, 11) // generates [1,11) --> 1-10 switch length { case 1: @@ -41,21 +42,25 @@ func RandomVarInt() uint64 { return 0 } +// VarIntLength returns the length of the variable integer when encoded as a var int func VarIntLength(v uint64) uint64 { buf := new(Buffer) EncodeVarInt(buf, v) return uint64(buf.Len()) } +// DecodeVarInt decodes a variable integer from the given data buffer. +// We use the algorithm used by Go, only BigEndian. func DecodeVarInt(data []byte) (uint64, []byte) { return DecodeVarIntGo(data) } +// EncodeVarInt encodes an integer as a variable int into the given data buffer. func EncodeVarInt(out *Buffer, v uint64) error { return EncodeVarIntGo(out, v) } -// Decode a variable integer from the given data buffer. +// DecodeVarIntGo decodes a variable integer from the given data buffer. // We use the algorithm used by Go, only BigEndian. func DecodeVarIntGo(data []byte) (uint64, []byte) { if data == nil || len(data) < 1 { @@ -74,7 +79,7 @@ func DecodeVarIntGo(data []byte) (uint64, []byte) { return v, data[cnt+1:] } -// Encode an integer as a variable int into the given data buffer. +// EncodeVarIntGo encodes an integer as a variable int into the given data buffer. func EncodeVarIntGo(out *Buffer, v uint64) error { if v == 0 { out.WriteByte(0) From 51ca3e381e93df9f82709c8966ee418c2a60a794 Mon Sep 17 00:00:00 2001 From: Michael Lam Date: Sat, 3 Aug 2019 10:57:15 -0700 Subject: [PATCH 2/6] 1) Add comments 2) change receivers 3) minor refactors --- common/primitives/hash.go | 71 +++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/common/primitives/hash.go b/common/primitives/hash.go index 36fbc54baf..2bc60eefbc 100644 --- a/common/primitives/hash.go +++ b/common/primitives/hash.go @@ -25,13 +25,16 @@ import ( "github.com/FactomProject/factomd/util/atomic" ) +// Hash is a convenient fixed []byte type created at the hash length type Hash [constants.HASH_LENGTH]byte +// The various interfaces Hash will be a part of, and must implement below var _ interfaces.Printable = (*Hash)(nil) var _ interfaces.IHash = (*Hash)(nil) var _ interfaces.BinaryMarshallableAndCopyable = (*Hash)(nil) var _ encoding.TextMarshaler = (*Hash)(nil) +// ZeroHash is a zero hash var ZeroHash interfaces.IHash = NewHash(constants.ZERO_HASH) var noRepeat map[string]int = make(map[string]int) @@ -46,35 +49,40 @@ func LogNilHashBug(msg string) { } +// IsHashNil returns true if receiver is nil, or the hash is zero func (h *Hash) IsHashNil() bool { return h == nil || reflect.ValueOf(h).IsNil() } + +// RandomHash returns a new random hash func RandomHash() interfaces.IHash { h := random.RandByteSliceOfLen(constants.HASH_LENGTH) - answer := new(Hash) - answer.SetBytes(h) + answer := NewHash(h) return answer } -func (c *Hash) Copy() (rval interfaces.IHash) { +// Copy returns a copy of this Hash +func (h *Hash) Copy() (rval interfaces.IHash) { defer func() { if rval != nil && reflect.ValueOf(rval).IsNil() { rval = nil // convert an interface that is nil to a nil interface LogNilHashBug("Hash.Copy() saw an interface that was nil") } }() - h := new(Hash) - err := h.SetBytes(c.Bytes()) + nh := new(Hash) + err := nh.SetBytes(h.Bytes()) if err != nil { panic(err) } - return h + return nh } -func (c *Hash) New() interfaces.BinaryMarshallableAndCopyable { +// New creates a new Hash (required for BinarymarshallableAndCopyable interface) +func (h *Hash) New() interfaces.BinaryMarshallableAndCopyable { return new(Hash) } +// MarshalText marshals the Hash as text func (h *Hash) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -84,6 +92,7 @@ func (h *Hash) MarshalText() (rval []byte, err error) { return []byte(hex.EncodeToString(h[:])), nil } +// IsZero returns true iff Hash is zero func (h *Hash) IsZero() bool { return h.String() == "0000000000000000000000000000000000000000000000000000000000000000" } @@ -100,6 +109,7 @@ func NewShaHashFromStr(hash string) (*Hash, error) { return h, nil } +// UnmarshalText unmarshals the input array func (h *Hash) UnmarshalText(b []byte) error { p, err := hex.DecodeString(string(b)) if err != nil { @@ -109,6 +119,7 @@ func (h *Hash) UnmarshalText(b []byte) error { return nil } +// Fixed returns the fixed []byte array func (h *Hash) Fixed() [constants.HASH_LENGTH]byte { // Might change the error produced by IHash in FD-398 if h == nil { @@ -117,11 +128,13 @@ func (h *Hash) Fixed() [constants.HASH_LENGTH]byte { return *h } +// PFixed returns a pointer to the fixed []byte array func (h *Hash) PFixed() *[constants.HASH_LENGTH]byte { // Might change the error produced by IHash in FD-398 return (*[constants.HASH_LENGTH]byte)(h) } +// Bytes returns a copy of the internal []byte array func (h *Hash) Bytes() (rval []byte) { defer func() { if r := recover(); r != nil { @@ -133,10 +146,12 @@ func (h *Hash) Bytes() (rval []byte) { return h.GetBytes() } +// GetHash is unused, merely here to implement the IHash interface func (Hash) GetHash() interfaces.IHash { return nil } +// CreateHash returns a hash created from all input entities func CreateHash(entities ...interfaces.BinaryMarshallable) (h interfaces.IHash, err error) { sha := sha256.New() h = new(Hash) @@ -151,6 +166,7 @@ func CreateHash(entities ...interfaces.BinaryMarshallable) (h interfaces.IHash, return } +// MarshalBinary returns a copy of the []byte array func (h *Hash) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -160,6 +176,7 @@ func (h *Hash) MarshalBinary() (rval []byte, err error) { return h.Bytes(), nil } +// UnmarshalBinaryData unmarshals the input array into the Hash func (h *Hash) UnmarshalBinaryData(p []byte) (newData []byte, err error) { defer func() { if r := recover(); r != nil { @@ -171,12 +188,13 @@ func (h *Hash) UnmarshalBinaryData(p []byte) (newData []byte, err error) { return } +// UnmarshalBinary unmarshals the input array into the Hash func (h *Hash) UnmarshalBinary(p []byte) (err error) { _, err = h.UnmarshalBinaryData(p) return } -// Make a copy of the hash in this hash. Changes to the return value WILL NOT be +// GetBytes makes a copy of the hash in this hash. Changes to the return value WILL NOT be // reflected in the source hash. You have to do a SetBytes to change the source // value. func (h *Hash) GetBytes() []byte { @@ -188,12 +206,12 @@ func (h *Hash) GetBytes() []byte { // SetBytes sets the bytes which represent the hash. An error is returned if // the number of bytes passed in is not constants.HASH_LENGTH. -func (hash *Hash) SetBytes(newHash []byte) error { +func (h *Hash) SetBytes(newHash []byte) error { nhlen := len(newHash) if nhlen != constants.HASH_LENGTH { return fmt.Errorf("invalid sha length of %v, want %v", nhlen, constants.HASH_LENGTH) } - copy(hash[:], newHash) + copy(h[:], newHash) return nil } @@ -208,7 +226,7 @@ func NewShaHash(newHash []byte) (*Hash, error) { return &sh, err } -// Create a Sha512[:256] Hash from a byte array +// Sha512Half creates a Sha512[:256] Hash from a byte array func Sha512Half(p []byte) (h *Hash) { sha := sha512.New() sha.Write(p) @@ -218,7 +236,7 @@ func Sha512Half(p []byte) (h *Hash) { return h } -// Convert a hash into a string with hex encoding +// String converts a hash into a hexidecimal (0-F) string func (h *Hash) String() string { if h == nil { return hex.EncodeToString(nil) @@ -227,10 +245,12 @@ func (h *Hash) String() string { } } +// ByteString returns the hash as a byte string func (h *Hash) ByteString() string { return string(h[:]) } +// HexToHash converts the input hexidecimal (0-F) string into the internal []byte array func HexToHash(hexStr string) (h interfaces.IHash, err error) { h = new(Hash) v, err := hex.DecodeString(hexStr) @@ -238,7 +258,7 @@ func HexToHash(hexStr string) (h interfaces.IHash, err error) { return h, err } -// Compare two Hashes +// IsSameAs compares two Hashes and returns true iff they hashs are binary identical func (a *Hash) IsSameAs(b interfaces.IHash) bool { if a == nil || b == nil { if a == nil && b == nil { @@ -254,7 +274,8 @@ func (a *Hash) IsSameAs(b interfaces.IHash) bool { return false } -// Is the hash a minute marker (the last byte indicates the minute number) +// IsMinuteMarker checks if the Hash is the hash of a minute marker (the last byte indicates the minute number) +// A minute marker is determined by having all but the last value as zero func (h *Hash) IsMinuteMarker() bool { if bytes.Equal(h[:constants.HASH_LENGTH-1], constants.ZERO_HASH[:constants.HASH_LENGTH-1]) { return true @@ -263,18 +284,22 @@ func (h *Hash) IsMinuteMarker() bool { return false } +// ToMinute returns the last byte of the Hash func (h *Hash) ToMinute() byte { return h[constants.HASH_LENGTH-1] } -func (e *Hash) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte returns the json encoded []byte array of the Hash +func (h *Hash) JSONByte() ([]byte, error) { + return EncodeJSON(h) } -func (e *Hash) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString returns the json encoded byte string of the Hash +func (h *Hash) JSONString() (string, error) { + return EncodeJSONString(h) } +// Loghashfixed logs the input hash /**************************************************************** DEBUG logging to keep full hash. Turned on from command line ****************************************************************/ @@ -319,6 +344,7 @@ func Loghashfixed(h [32]byte) { } } +// Loghash logs the input interface func Loghash(h interfaces.IHash) { if h == nil { return @@ -330,7 +356,7 @@ func Loghash(h interfaces.IHash) { * Support functions **********************/ -// Create a Sha256 Hash from a byte array +// Sha creates a Sha256 Hash from a byte array func Sha(p []byte) interfaces.IHash { h := new(Hash) b := sha256.Sum256(p) @@ -339,7 +365,7 @@ func Sha(p []byte) interfaces.IHash { return h } -// Shad Double Sha256 Hash; sha256(sha256(data)) +// Shad returns a new hash created by double hashing the input: Double Sha256 Hash; sha256(sha256(data)) func Shad(data []byte) interfaces.IHash { h1 := sha256.Sum256(data) h2 := sha256.Sum256(h1[:]) @@ -348,18 +374,20 @@ func Shad(data []byte) interfaces.IHash { return h } +// NewZeroHash creates a new zero hash object func NewZeroHash() interfaces.IHash { h := new(Hash) return h } +// NewHash creates a new object for the input hash func NewHash(b []byte) interfaces.IHash { h := new(Hash) h.SetBytes(b) return h } -// shad Double Sha256 Hash; sha256(sha256(data)) +// DoubleSha returns a new hash created by double hashing the input: shad Double Sha256 Hash; sha256(sha256(data)) func DoubleSha(data []byte) []byte { h1 := sha256.Sum256(data) h2 := sha256.Sum256(h1[:]) @@ -367,6 +395,7 @@ func DoubleSha(data []byte) []byte { return h2[:] } +// NewShaHashFromStruct marshals the input struct into a json byte array, then double hashes the json array func NewShaHashFromStruct(DataStruct interface{}) (interfaces.IHash, error) { jsonbytes, err := json.Marshal(DataStruct) if err != nil { From 18df2ad06b9feb6180e03e872959a7d39c935635 Mon Sep 17 00:00:00 2001 From: Michael Lam Date: Sat, 3 Aug 2019 10:57:44 -0700 Subject: [PATCH 3/6] 1) Add comments 2) remove extraneous error checks --- common/primitives/buffer.go | 66 ++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/common/primitives/buffer.go b/common/primitives/buffer.go index 3ad2943c33..649ad76f0d 100644 --- a/common/primitives/buffer.go +++ b/common/primitives/buffer.go @@ -14,16 +14,19 @@ import ( "github.com/FactomProject/goleveldb/leveldb/errors" ) +// Buffer contains a bytes.Buffer type Buffer struct { bytes.Buffer } +// DeepCopyBytes returns the remainder of the unread buffer. Despite its name, it DOES NOT COPY! func (b *Buffer) DeepCopyBytes() []byte { // Despite the name this purposefully does not copy, copying turns out to blow up memory when unmarshalling // because the []bytes is very big with many messages and it all gets copied many many times return b.Next(b.Len()) } +// NewBuffer copies the input []byte array int a new Buffer object and returns the Buffer func NewBuffer(buf []byte) *Buffer { tmp := new(Buffer) c := make([]byte, len(buf)) @@ -32,18 +35,17 @@ func NewBuffer(buf []byte) *Buffer { return tmp } +// PeekByte returns the next unread byte in the buffer without advancing the read state func (b *Buffer) PeekByte() (byte, error) { by, err := b.ReadByte() if err != nil { return by, err } err = b.UnreadByte() - if err != nil { - return by, err - } - return by, nil + return by, err } +// PushBinaryMarshallableMsgArray marshals the input message array and writes it into the Buffer func (b *Buffer) PushBinaryMarshallableMsgArray(bm []interfaces.IMsg) error { err := b.PushInt(len(bm)) if err != nil { @@ -59,6 +61,7 @@ func (b *Buffer) PushBinaryMarshallableMsgArray(bm []interfaces.IMsg) error { return nil } +// PushBinaryMarshallable marshals the input and writes it to the Buffer func (b *Buffer) PushBinaryMarshallable(bm interfaces.BinaryMarshallable) error { if bm == nil || reflect.ValueOf(bm).IsNil() { return fmt.Errorf("BinaryMarshallable is nil") @@ -68,20 +71,21 @@ func (b *Buffer) PushBinaryMarshallable(bm interfaces.BinaryMarshallable) error return err } _, err = b.Write(bin) - if err != nil { - return err - } - return nil + return err + } +// PushMsg marshals and writes the input message to the Buffer func (b *Buffer) PushMsg(msg interfaces.IMsg) error { return b.PushBinaryMarshallable(msg) } +// PushString marshals and writes the string to the Buffer func (b *Buffer) PushString(s string) error { return b.PushBytes([]byte(s)) } +// PushBytes marshals and writes the []byte array to the Buffer func (b *Buffer) PushBytes(h []byte) error { l := uint64(len(h)) @@ -91,33 +95,32 @@ func (b *Buffer) PushBytes(h []byte) error { } _, err = b.Write(h) - if err != nil { - return err - } - - return nil + return err } +// PushIHash marshals and writes the input hash to the Buffer func (b *Buffer) PushIHash(h interfaces.IHash) error { return b.PushBytes(h.Bytes()) } +// Push appends the input []byte array to the Buffer. Return error will always +// be nil, because Write gaurantees a nil error return. func (b *Buffer) Push(h []byte) error { _, err := b.Write(h) - if err != nil { - return err - } - return nil + return err } +// PushUInt32 writes the input uint32 to the Buffer func (b *Buffer) PushUInt32(i uint32) error { return binary.Write(b, binary.BigEndian, &i) } +// PushUInt64 writes the input uint64 to the Buffer func (b *Buffer) PushUInt64(i uint64) error { return binary.Write(b, binary.BigEndian, &i) } +// PushBool writes the input bool to the Buffer func (b *Buffer) PushBool(boo bool) error { var err error if boo { @@ -128,30 +131,37 @@ func (b *Buffer) PushBool(boo bool) error { return err } +// PushTimestamp writes a timestamp into the Buffer func (b *Buffer) PushTimestamp(ts interfaces.Timestamp) error { return b.PushInt64(ts.GetTimeMilli()) } +// PushVarInt writes the smallest possible data to the Buffer to represent the input integer func (b *Buffer) PushVarInt(vi uint64) error { return EncodeVarInt(b, vi) } +// PushByte writes the input byte to the Buffer. Returned error is always nil. func (b *Buffer) PushByte(h byte) error { return b.WriteByte(h) } +// PushInt64 writes the input int64 to the Buffer func (b *Buffer) PushInt64(i int64) error { return b.PushUInt64(uint64(i)) } +// PushUInt8 writes the input int8 to the Buffer func (b *Buffer) PushUInt8(h uint8) error { return b.PushByte(byte(h)) } +// PushUInt16 writes the input uint16 to the Buffer func (b *Buffer) PushUInt16(i uint16) error { return binary.Write(b, binary.BigEndian, &i) } +// PopUInt16 reads a uint16 from the Buffer func (b *Buffer) PopUInt16() (uint16, error) { var i uint16 err := binary.Read(b, binary.BigEndian, &i) @@ -161,6 +171,7 @@ func (b *Buffer) PopUInt16() (uint16, error) { return i, nil } +// PopUInt8 reads a uint8 from the Buffer func (b *Buffer) PopUInt8() (uint8, error) { h, err := b.PopByte() if err != nil { @@ -169,18 +180,21 @@ func (b *Buffer) PopUInt8() (uint8, error) { return uint8(h), nil } +// PushInt writes an int to the Buffer. func (b *Buffer) PushInt(i int) error { - return b.PushInt64(int64(i)) + return b.PushInt64(int64(i)) // Safe to cast to int64 even on 32 bit systems } +// PopInt reads an int from the Buffer func (b *Buffer) PopInt() (int, error) { i, err := b.PopInt64() if err != nil { return 0, err } - return int(i), nil + return int(i), nil // Safe to cast int64 to int on 32 bit systems iff undoing PushInt } +// PopInt64 reads an int64 from the Buffer func (b *Buffer) PopInt64() (int64, error) { i, err := b.PopUInt64() if err != nil { @@ -189,10 +203,12 @@ func (b *Buffer) PopInt64() (int64, error) { return int64(i), nil } +// PopByte reads a byte from the buffer func (b *Buffer) PopByte() (byte, error) { return b.ReadByte() } +// PopVarInt reads an integer from the Buffer func (b *Buffer) PopVarInt() (uint64, error) { h := b.DeepCopyBytes() l, rest := DecodeVarInt(h) @@ -204,6 +220,7 @@ func (b *Buffer) PopVarInt() (uint64, error) { return l, nil } +// PopUInt32 reads a uint32 from the Buffer func (b *Buffer) PopUInt32() (uint32, error) { var i uint32 err := binary.Read(b, binary.BigEndian, &i) @@ -213,6 +230,7 @@ func (b *Buffer) PopUInt32() (uint32, error) { return i, nil } +// PopUInt64 reads a uint64 from the Buffer func (b *Buffer) PopUInt64() (uint64, error) { var i uint64 err := binary.Read(b, binary.BigEndian, &i) @@ -222,6 +240,7 @@ func (b *Buffer) PopUInt64() (uint64, error) { return i, nil } +// PopBool reads a bool from the Buffer func (b *Buffer) PopBool() (bool, error) { boo, err := b.ReadByte() if err != nil { @@ -230,6 +249,7 @@ func (b *Buffer) PopBool() (bool, error) { return boo > 0, nil } +// PopTimestamp reads a time stamp from the Buffer func (b *Buffer) PopTimestamp() (interfaces.Timestamp, error) { ts, err := b.PopInt64() if err != nil { @@ -238,6 +258,7 @@ func (b *Buffer) PopTimestamp() (interfaces.Timestamp, error) { return NewTimestampFromMilliseconds(uint64(ts)), nil } +// PopString reads a string from the Buffer func (b *Buffer) PopString() (string, error) { h, err := b.PopBytes() if err != nil { @@ -246,6 +267,7 @@ func (b *Buffer) PopString() (string, error) { return fmt.Sprintf("%s", h), nil } +// PopBytes reads a byte array from the Buffer func (b *Buffer) PopBytes() ([]byte, error) { l, err := b.PopVarInt() if err != nil || int(l) < 0 { @@ -263,6 +285,7 @@ func (b *Buffer) PopBytes() ([]byte, error) { return answer, nil } +// PopIHash reads an hash from the Buffer func (b *Buffer) PopIHash() (interfaces.IHash, error) { bb, err := b.PopBytes() if err != nil { @@ -271,6 +294,7 @@ func (b *Buffer) PopIHash() (interfaces.IHash, error) { return NewHash(bb), nil } +// PopLen reads a number of bytes equal to the input length from the Buffer func (b *Buffer) PopLen(l int) ([]byte, error) { answer := make([]byte, l) _, err := b.Read(answer) @@ -280,6 +304,7 @@ func (b *Buffer) PopLen(l int) ([]byte, error) { return answer, nil } +// Pop reads a number of bytes equal to the length of the input []byte array func (b *Buffer) Pop(h []byte) error { _, err := b.Read(h) if err != nil { @@ -288,6 +313,7 @@ func (b *Buffer) Pop(h []byte) error { return nil } +// PopBinaryMarshallable reads a binary marshallable interface object from the Buffer func (b *Buffer) PopBinaryMarshallable(dst interfaces.BinaryMarshallable) error { if dst == nil { return fmt.Errorf("Destination is nil") @@ -306,6 +332,7 @@ func (b *Buffer) PopBinaryMarshallable(dst interfaces.BinaryMarshallable) error return nil } +// PopBinaryMarshallableMsgArray reads a message array from the Buffer func (b *Buffer) PopBinaryMarshallableMsgArray() ([]interfaces.IMsg, error) { l, err := b.PopInt() if err != nil { @@ -327,6 +354,7 @@ func (b *Buffer) PopBinaryMarshallableMsgArray() ([]interfaces.IMsg, error) { var General interfaces.IGeneralMsg +// PopMsg reads a message from the Buffer func (b *Buffer) PopMsg() (msg interfaces.IMsg, err error) { h := b.DeepCopyBytes() rest, msg, err := General.UnmarshalMessageData(h) From c09938bc68de1a6622008213328490853ed2158b Mon Sep 17 00:00:00 2001 From: Michael Lam Date: Sat, 3 Aug 2019 10:58:14 -0700 Subject: [PATCH 4/6] 1) Add comments 2) Add minor tests for isSameAs() --- common/primitives/binary_test.go | 80 ++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/common/primitives/binary_test.go b/common/primitives/binary_test.go index 6f05a16f19..89ac38db52 100644 --- a/common/primitives/binary_test.go +++ b/common/primitives/binary_test.go @@ -17,6 +17,7 @@ import ( var testBytes []byte var testStr string = "00010203040506070809101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263" +// init initializes the testBytes variable above func init() { h, err := hex.DecodeString(testStr) if err != nil { @@ -26,6 +27,7 @@ func init() { testBytes = h } +// TestUnmarshalNilByteSlice32 tests that unmarshaling catches inappropriate empty inputs func TestUnmarshalNilByteSlice32(t *testing.T) { defer func() { if r := recover(); r != nil { @@ -45,6 +47,7 @@ func TestUnmarshalNilByteSlice32(t *testing.T) { } } +// TestUnmarshalNilByteSlice64 tests that unmarshaling catches inappropriate empty inputs func TestUnmarshalNilByteSlice64(t *testing.T) { defer func() { if r := recover(); r != nil { @@ -64,6 +67,7 @@ func TestUnmarshalNilByteSlice64(t *testing.T) { } } +// TestUnmarshalNilByteSlice6 tests that unmarshaling catches inappropriate empty inputs func TestUnmarshalNilByteSlice6(t *testing.T) { defer func() { if r := recover(); r != nil { @@ -83,6 +87,7 @@ func TestUnmarshalNilByteSlice6(t *testing.T) { } } +// TestUnmarshalNilByteSliceSig tests that unmarshaling catches inappropriate empty inputs func TestUnmarshalNilByteSliceSig(t *testing.T) { defer func() { if r := recover(); r != nil { @@ -102,6 +107,7 @@ func TestUnmarshalNilByteSliceSig(t *testing.T) { } } +// TestUnmarshalNilByteSlice20 tests that unmarshaling catches inappropriate empty inputs func TestUnmarshalNilByteSlice20(t *testing.T) { defer func() { if r := recover(); r != nil { @@ -121,6 +127,11 @@ func TestUnmarshalNilByteSlice20(t *testing.T) { } } +// TestAreBytesEqual checks the AreBytesEqual fuction with N random ByteSlices in a variety of ways: +// 1) ByteSlice when copied and checked against each other, are in fact equal +// 2) Small perturbations to the copied ByteSlice are correctly returned as not equal +// 3) nil comparisons to the ByteSlice return not equal +// Finally, checks nil to nil to ensure returns equal func TestAreBytesEqual(t *testing.T) { for i := 0; i < 1000; i++ { b1 := random.RandByteSlice() @@ -162,6 +173,11 @@ func TestAreBytesEqual(t *testing.T) { } } +// TestAreBinaryMarshallablesEqual checks N random hashes in a variety of ways: +// 1) hashes when copied and checked against each other and themselves, are in fact equal +// 2) different random hashes are correctly returned as not equal +// 3) nil comparisons to the hash return not equal +// Finally, checks nil to nil to ensure returns equal func TestAreBinaryMarshallablesEqual(t *testing.T) { for i := 0; i < 1000; i++ { h1 := RandomHash() @@ -224,6 +240,7 @@ func TestAreBinaryMarshallablesEqual(t *testing.T) { } } +// TestEncodeBinary checks N random []byte arrays can be encoded and decoded properly func TestEncodeBinary(t *testing.T) { for i := 0; i < 1000; i++ { h1 := random.RandByteSlice() @@ -243,6 +260,7 @@ func TestEncodeBinary(t *testing.T) { } } +// TestStringToByteSlice32 checks that N random strings can be saved into the ByteSlice32 and recalled func TestStringToByteSlice32(t *testing.T) { for i := 0; i < 1000; i++ { h := random.RandByteSliceOfLen(32) @@ -251,9 +269,17 @@ func TestStringToByteSlice32(t *testing.T) { if b.String() != s { t.Errorf("Invalid BS32 parsed") } + b2 := b + if b.IsSameAs(b2) == false { + t.Errorf("ByteSlice32 IsSameAs could not detect same input") + } + if b.IsSameAs(nil) == true { + t.Errorf("ByteSlice32 IsSameAs error on nil input ") + } } } +// TestByte32ToByteSlice32 checks that N random []byte arrays can be converted to ByteSlice32 func TestByte32ToByteSlice32(t *testing.T) { for i := 0; i < 1000; i++ { h := random.RandByteSliceOfLen(32) @@ -266,6 +292,7 @@ func TestByte32ToByteSlice32(t *testing.T) { } } +// TestByteSliceSig checks that N random ByteSliceSigs can be marshalled and unmarshalled appropriately func TestByteSliceSig(t *testing.T) { for i := 0; i < 1000; i++ { bss := new(ByteSliceSig) @@ -292,6 +319,15 @@ func TestByteSliceSig(t *testing.T) { t.Errorf("Equal bytes are not equal") } + bssame := new(ByteSliceSig) + _ = bssame.UnmarshalBinary(b1) // Shouldn't throw error because already proven unmarshalable above + if bssame.IsSameAs(bss) == false { + t.Errorf("ByteSliceSig IsSameAs could not detect same input") + } + if bssame.IsSameAs(nil) == true { + t.Errorf("ByteSliceSig IsSameAs error on nil input ") + } + extra := random.RandByteSlice() b3 := append(b1, extra...) @@ -320,6 +356,7 @@ func TestByteSliceSig(t *testing.T) { } } +// TestByteSlice20 checks that N random ByteSlice20 can be marshalled and unmarshalled appropriately func TestByteSlice20(t *testing.T) { for i := 0; i < 1000; i++ { bss := new(ByteSlice20) @@ -346,6 +383,15 @@ func TestByteSlice20(t *testing.T) { t.Errorf("Equal bytes are not equal") } + bssame := new(ByteSlice20) + _ = bssame.UnmarshalBinary(b1) // Shouldn't throw error because already proven unmarshalable above + if bssame.IsSameAs(bss) == false { + t.Errorf("ByteSlice20 IsSameAs could not detect same input") + } + if bssame.IsSameAs(nil) == true { + t.Errorf("ByteSlice20 IsSameAs error on nil input ") + } + extra := random.RandByteSlice() b3 := append(b1, extra...) @@ -363,6 +409,7 @@ func TestByteSlice20(t *testing.T) { } } +// TestByteSlice checks that N random ByteSlice can be marshalled and unmarshalled appropriately func TestByteSlice(t *testing.T) { for i := 0; i < 1000; i++ { bss := new(ByteSlice) @@ -381,6 +428,15 @@ func TestByteSlice(t *testing.T) { t.Errorf("Equal bytes are not equal") } + bssame := new(ByteSlice) + _ = bssame.UnmarshalBinary(b1) // Shouldn't throw error because already proven unmarshalable above + if bssame.IsSameAs(bss) == false { + t.Errorf("ByteSlice IsSameAs could not detect same input") + } + if bssame.IsSameAs(nil) == true { + t.Errorf("ByteSlice IsSameAs error on nil input ") + } + extra := random.RandByteSlice() b3 := append(b1, extra...) @@ -400,6 +456,7 @@ func TestByteSlice(t *testing.T) { } } +// TestBA64Misc checks that ByteSlice64 can be marshalled and unmarshalled appropriately func TestBA64Misc(t *testing.T) { ba := new(ByteSlice64) @@ -435,8 +492,16 @@ func TestBA64Misc(t *testing.T) { if json != "\""+testStr+"\"" { t.Errorf("Failed JSONString - %s", json) } + ba2 := ba + if ba.IsSameAs(ba2) == false { + t.Errorf("ByteSlice64 IsSameAs could not detect same input") + } + if ba.IsSameAs(nil) == true { + t.Errorf("ByteSlice64 IsSameAs error on nil input ") + } } +// TestBA32Misc checks that ByteSlice32 can be marshalled and unmarshalled appropriately func TestBA32Misc(t *testing.T) { ba := new(ByteSlice32) testStr32 := testStr[:64] @@ -474,8 +539,16 @@ func TestBA32Misc(t *testing.T) { if json != "\""+testStr32+"\"" { t.Errorf("Failed JSONString - %s", json) } + ba2 := ba + if ba.IsSameAs(ba2) == false { + t.Errorf("ByteSlice32 IsSameAs could not detect same input") + } + if ba.IsSameAs(nil) == true { + t.Errorf("ByteSlice32 IsSameAs error on nil input ") + } } +// TestBA6Misc checks that ByteSlice6 can be marshalled and unmarshalled appropriately func TestBA6Misc(t *testing.T) { ba := new(ByteSlice6) testStr6 := testStr[:12] @@ -513,4 +586,11 @@ func TestBA6Misc(t *testing.T) { if json != "\""+testStr6+"\"" { t.Errorf("Failed JSONString - %s", json) } + ba2 := ba + if ba.IsSameAs(ba2) == false { + t.Errorf("ByteSlice6 IsSameAs could not detect same input") + } + if ba.IsSameAs(nil) == true { + t.Errorf("ByteSlice6 IsSameAs error on nil input ") + } } From 5134ffd400f5496e3b30948e2d64dd89c92d7c64 Mon Sep 17 00:00:00 2001 From: Michael Lam Date: Sat, 3 Aug 2019 10:58:44 -0700 Subject: [PATCH 5/6] 1) Fix receivers 2) add comments 3) refactor unmarshal function --- common/primitives/binary.go | 243 ++++++++++++++++++++---------------- 1 file changed, 138 insertions(+), 105 deletions(-) diff --git a/common/primitives/binary.go b/common/primitives/binary.go index e5ee035743..a0895bce91 100644 --- a/common/primitives/binary.go +++ b/common/primitives/binary.go @@ -14,6 +14,7 @@ import ( "github.com/FactomProject/factomd/common/primitives/random" ) +// AreBytesEqual returns true iff the lengths and byte values of the input []byte arrays are equal func AreBytesEqual(b1, b2 []byte) bool { if len(b1) != len(b2) { return false @@ -26,6 +27,8 @@ func AreBytesEqual(b1, b2 []byte) bool { return true } +// AreBinaryMarshallablesEqual returns true if the input interfaces are both nil, or both exist +// and can be marshalled into byte identical arrays func AreBinaryMarshallablesEqual(b1, b2 interfaces.BinaryMarshallable) (bool, error) { if b1 == nil { if b2 == nil { @@ -48,19 +51,38 @@ func AreBinaryMarshallablesEqual(b1, b2 interfaces.BinaryMarshallable) (bool, er return AreBytesEqual(bytes1, bytes2), nil } +// UnmarshalBinaryDataOfLength unmarshals an arbitrary input []byte array up to specified length, returning residual +func UnmarshalBinaryDataOfLength(dest []byte, source []byte, length int) (newData []byte, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("Error unmarshalling: %v", r) + } + }() + if (source == nil && length > 0) || len(source) < length { + return nil, fmt.Errorf("Not enough data to unmarshal") + } + copy(dest[:], source[:length]) + newData = source[length:] + return +} + +// EncodeBinary returns the hexadecimal (0-F) encoding of input byte array func EncodeBinary(bytes []byte) string { return hex.EncodeToString(bytes) } +// DecodeBinary returns a byte array of the decoded input hexadecimal (0-F) string func DecodeBinary(bytes string) ([]byte, error) { return hex.DecodeString(bytes) } +// ByteSlice32 is a fixed [32]byte array type ByteSlice32 [32]byte var _ interfaces.Printable = (*ByteSlice32)(nil) var _ interfaces.BinaryMarshallable = (*ByteSlice32)(nil) +// StringToByteSlice32 converts the input hexidecimal string (0-F) to a new ByteSlice32 func StringToByteSlice32(s string) *ByteSlice32 { bin, err := DecodeBinary(s) if err != nil { @@ -74,6 +96,7 @@ func StringToByteSlice32(s string) *ByteSlice32 { return bs } +// Byte32ToByteSlice32 returns a new ByteSlice32 containing the input data func Byte32ToByteSlice32(b [32]byte) *ByteSlice32 { bs := new(ByteSlice32) err := bs.UnmarshalBinary(b[:]) @@ -83,16 +106,18 @@ func Byte32ToByteSlice32(b [32]byte) *ByteSlice32 { return bs } -func (a *ByteSlice32) IsSameAs(b *ByteSlice32) bool { - if a == nil || b == nil { - if a == nil && b == nil { +// IsSameAs returns true iff input ByteSlice32 is binary identical to this ByteSlice +func (bs *ByteSlice32) IsSameAs(b *ByteSlice32) bool { + if bs == nil || b == nil { + if bs == nil && b == nil { return true } return false } - return AreBytesEqual(a[:], b[:]) + return AreBytesEqual(bs[:], b[:]) } +// MarshalBinary marshals this ByteSlice32 into []byte array func (bs *ByteSlice32) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -102,41 +127,39 @@ func (bs *ByteSlice32) MarshalBinary() (rval []byte, err error) { return bs[:], nil } +// Fixed returns the internal fixed byte array data func (bs *ByteSlice32) Fixed() [32]byte { return *bs } +// UnmarshalBinaryData unmarshals the input data into this ByteSlice32 func (bs *ByteSlice32) UnmarshalBinaryData(data []byte) (newData []byte, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Error unmarshalling: %v", r) - } - }() - if data == nil || len(data) < 32 { - return nil, fmt.Errorf("Not enough data to unmarshal") - } - copy(bs[:], data[:32]) - newData = data[32:] + newData, err = UnmarshalBinaryDataOfLength(bs[:], data, 32) return } +// UnmarshalBinary unmarshals the input data into this ByteSlice32 func (bs *ByteSlice32) UnmarshalBinary(data []byte) (err error) { _, err = bs.UnmarshalBinaryData(data) return } -func (e *ByteSlice32) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte returns the encoded json data of this ByteSlice32 +func (bs *ByteSlice32) JSONByte() ([]byte, error) { + return EncodeJSON(bs) } -func (e *ByteSlice32) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString returns the encoded json byte string of this ByteSlice32 +func (bs *ByteSlice32) JSONString() (string, error) { + return EncodeJSONString(bs) } +// String returns the hexidecimal string (0-F) of this ByteSlice32 func (bs *ByteSlice32) String() string { return fmt.Sprintf("%x", bs[:]) } +// MarshalText marshals this ByteSlice32 into the returned array func (bs *ByteSlice32) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -146,21 +169,24 @@ func (bs *ByteSlice32) MarshalText() (rval []byte, err error) { return []byte(bs.String()), nil } +// ByteSlice64 is a fixed [64]byte array type ByteSlice64 [64]byte var _ interfaces.Printable = (*ByteSlice64)(nil) var _ interfaces.BinaryMarshallable = (*ByteSlice64)(nil) -func (a *ByteSlice64) IsSameAs(b *ByteSlice64) bool { - if a == nil || b == nil { - if a == nil && b == nil { +// IsSameAs returns true iff the input ByteSlice64 is binary identical to this ByteSlice64 +func (bs *ByteSlice64) IsSameAs(b *ByteSlice64) bool { + if bs == nil || b == nil { + if bs == nil && b == nil { return true } return false } - return AreBytesEqual(a[:], b[:]) + return AreBytesEqual(bs[:], b[:]) } +// MarshalBinary marshals this ByteSlice64 func (bs *ByteSlice64) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -170,37 +196,34 @@ func (bs *ByteSlice64) MarshalBinary() (rval []byte, err error) { return bs[:], nil } +// UnmarshalBinaryData unmarshals the input data into this ByteSlice64 func (bs *ByteSlice64) UnmarshalBinaryData(data []byte) (newData []byte, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Error unmarshalling: %v", r) - } - }() - if data == nil || len(data) < 64 { - return nil, fmt.Errorf("Not enough data to unmarshal") - } - copy(bs[:], data[:64]) - newData = data[64:] + newData, err = UnmarshalBinaryDataOfLength(bs[:], data, 64) return } +// UnmarshalBinary unmarshals the input data into this ByteSlice64 func (bs *ByteSlice64) UnmarshalBinary(data []byte) (err error) { _, err = bs.UnmarshalBinaryData(data) return } -func (e *ByteSlice64) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte returns the json encoded data +func (bs *ByteSlice64) JSONByte() ([]byte, error) { + return EncodeJSON(bs) } -func (e *ByteSlice64) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString returns the json encoded byte string +func (bs *ByteSlice64) JSONString() (string, error) { + return EncodeJSONString(bs) } +// String returns the json encoded hexidecimal (0-F) string func (bs *ByteSlice64) String() string { return fmt.Sprintf("%x", bs[:]) } +// MarshalText marshals this ByteSlice64 into text func (bs *ByteSlice64) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -210,21 +233,24 @@ func (bs *ByteSlice64) MarshalText() (rval []byte, err error) { return []byte(bs.String()), nil } +// ByteSlice6 is a fixed [6]byte array type ByteSlice6 [6]byte var _ interfaces.Printable = (*ByteSlice6)(nil) var _ interfaces.BinaryMarshallable = (*ByteSlice6)(nil) -func (a *ByteSlice6) IsSameAs(b *ByteSlice6) bool { - if a == nil || b == nil { - if a == nil && b == nil { +// IsSameAs returns true iff the input ByteSlice6 is binary identical to this ByteSlice6 +func (bs *ByteSlice6) IsSameAs(b *ByteSlice6) bool { + if bs == nil || b == nil { + if bs == nil && b == nil { return true } return false } - return AreBytesEqual(a[:], b[:]) + return AreBytesEqual(bs[:], b[:]) } +// MarshalBinary marshals this ByteSlice6 into a []byte array func (bs *ByteSlice6) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -234,37 +260,34 @@ func (bs *ByteSlice6) MarshalBinary() (rval []byte, err error) { return bs[:], nil } +// UnmarshalBinaryData unmarshals the input data into the ByteSlice6 func (bs *ByteSlice6) UnmarshalBinaryData(data []byte) (newData []byte, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Error unmarshalling: %v", r) - } - }() - if data == nil || len(data) < 6 { - return nil, fmt.Errorf("Not enough data to unmarshal") - } - copy(bs[:], data[:6]) - newData = data[6:] + newData, err = UnmarshalBinaryDataOfLength(bs[:], data, 6) return } +// UnmarshalBinary unmarshals the input data into the ByteSlice6 func (bs *ByteSlice6) UnmarshalBinary(data []byte) (err error) { _, err = bs.UnmarshalBinaryData(data) return } -func (e *ByteSlice6) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte returns the json encoded data of the ByteSlice6 +func (bs *ByteSlice6) JSONByte() ([]byte, error) { + return EncodeJSON(bs) } -func (e *ByteSlice6) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString returns the json encoded byte string of the ByteSlice6 +func (bs *ByteSlice6) JSONString() (string, error) { + return EncodeJSONString(bs) } +// String returns the hexidecimal (0-F) string of the ByteSlice6S func (bs *ByteSlice6) String() string { return fmt.Sprintf("%x", bs[:]) } +// MarshalText marshals the ByteSlice6 into text func (bs *ByteSlice6) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -274,21 +297,24 @@ func (bs *ByteSlice6) MarshalText() (rval []byte, err error) { return []byte(bs.String()), nil } +// ByteSliceSig is a fixed byte array of the ed25519 signature length type ByteSliceSig [ed25519.SignatureSize]byte var _ interfaces.Printable = (*ByteSliceSig)(nil) var _ interfaces.BinaryMarshallable = (*ByteSliceSig)(nil) -func (a *ByteSliceSig) IsSameAs(b *ByteSliceSig) bool { - if a == nil || b == nil { - if a == nil && b == nil { +// IsSameAs returns true iff the input ByteSliceSig is binary identical to this ByteSliceSig +func (bs *ByteSliceSig) IsSameAs(b *ByteSliceSig) bool { + if bs == nil || b == nil { + if bs == nil && b == nil { return true } return false } - return AreBytesEqual(a[:], b[:]) + return AreBytesEqual(bs[:], b[:]) } +// MarshalBinary marshals this ByteSliceSig into []byte array func (bs *ByteSliceSig) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -298,6 +324,7 @@ func (bs *ByteSliceSig) MarshalBinary() (rval []byte, err error) { return bs[:], nil } +// GetFixed returns a new copy of the internal byte array func (bs *ByteSliceSig) GetFixed() ([ed25519.SignatureSize]byte, error) { answer := [ed25519.SignatureSize]byte{} copy(answer[:], bs[:]) @@ -305,20 +332,13 @@ func (bs *ByteSliceSig) GetFixed() ([ed25519.SignatureSize]byte, error) { return answer, nil } +// UnmarshalBinaryData unmarshals the input data into the ByteSliceSig func (bs *ByteSliceSig) UnmarshalBinaryData(data []byte) (newData []byte, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Error unmarshalling: %v", r) - } - }() - if data == nil || len(data) < ed25519.SignatureSize { - return nil, fmt.Errorf("Not enough data to unmarshal") - } - copy(bs[:], data[:ed25519.SignatureSize]) - newData = data[ed25519.SignatureSize:] + newData, err = UnmarshalBinaryDataOfLength(bs[:], data, ed25519.SignatureSize) return } +// UnmarshalBinary unmarshals the input data into the ByteSliceSig func (bs *ByteSliceSig) UnmarshalBinary(data []byte) (err error) { if len(data) < ed25519.SignatureSize { return fmt.Errorf("Byte slice too short to unmarshal") @@ -327,18 +347,22 @@ func (bs *ByteSliceSig) UnmarshalBinary(data []byte) (err error) { return } -func (e *ByteSliceSig) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte returns the json encoded data of the ByteSliceSig +func (bs *ByteSliceSig) JSONByte() ([]byte, error) { + return EncodeJSON(bs) } -func (e *ByteSliceSig) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString returns the json encoded byte string of the ByteSliceSig +func (bs *ByteSliceSig) JSONString() (string, error) { + return EncodeJSONString(bs) } +// String returns a hexidecimal (0-F) string of the ByteSliceSig func (bs *ByteSliceSig) String() string { return fmt.Sprintf("%x", bs[:]) } +// MarshalText marshals the ByteSliceSig into text func (bs *ByteSliceSig) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -348,6 +372,7 @@ func (bs *ByteSliceSig) MarshalText() (rval []byte, err error) { return []byte(bs.String()), nil } +// UnmarshalText unmarshals the input text to the ByteSliceSig func (bs *ByteSliceSig) UnmarshalText(text []byte) error { b, err := hex.DecodeString(string(text)) if err != nil { @@ -356,21 +381,24 @@ func (bs *ByteSliceSig) UnmarshalText(text []byte) error { return bs.UnmarshalBinary(b) } +// ByteSlice20 is a fixed [20]byte type ByteSlice20 [20]byte var _ interfaces.Printable = (*ByteSlice20)(nil) var _ interfaces.BinaryMarshallable = (*ByteSlice20)(nil) -func (a *ByteSlice20) IsSameAs(b *ByteSlice20) bool { - if a == nil || b == nil { - if a == nil && b == nil { +// IsSameAs returns true iff input ByteSlice20 is binary identical to this ByteSlice20 +func (bs *ByteSlice20) IsSameAs(b *ByteSlice20) bool { + if bs == nil || b == nil { + if bs == nil && b == nil { return true } return false } - return AreBytesEqual(a[:], b[:]) + return AreBytesEqual(bs[:], b[:]) } +// MarshalBinary returns the byte array of the ByteSlice20 func (bs *ByteSlice20) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -380,6 +408,7 @@ func (bs *ByteSlice20) MarshalBinary() (rval []byte, err error) { return bs[:], nil } +// GetFixed returns a new copy of the internal byte array func (bs *ByteSlice20) GetFixed() ([20]byte, error) { answer := [20]byte{} copy(answer[:], bs[:]) @@ -387,37 +416,34 @@ func (bs *ByteSlice20) GetFixed() ([20]byte, error) { return answer, nil } +// UnmarshalBinaryData unmarshals the input data into the ByteSlice20 func (bs *ByteSlice20) UnmarshalBinaryData(data []byte) (newData []byte, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Error unmarshalling: %v", r) - } - }() - if data == nil || len(data) < 20 { - return nil, fmt.Errorf("Not enough data to unmarshal") - } - copy(bs[:], data[:20]) - newData = data[20:] + newData, err = UnmarshalBinaryDataOfLength(bs[:], data, 20) return } +// UnmarshalBinary unmarshals the input data into the ByteSlice20 func (bs *ByteSlice20) UnmarshalBinary(data []byte) (err error) { _, err = bs.UnmarshalBinaryData(data) return } -func (e *ByteSlice20) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte returns the encoded JSON format data +func (bs *ByteSlice20) JSONByte() ([]byte, error) { + return EncodeJSON(bs) } -func (e *ByteSlice20) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString returns the encoded JSON format byte string +func (bs *ByteSlice20) JSONString() (string, error) { + return EncodeJSONString(bs) } +// String returns a hexidecimal (0-F) string of the internal data func (bs *ByteSlice20) String() string { return fmt.Sprintf("%x", bs[:]) } +// MarshalText marshals the internal ByteSlice20 func (bs *ByteSlice20) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -427,6 +453,7 @@ func (bs *ByteSlice20) MarshalText() (rval []byte, err error) { return []byte(bs.String()), nil } +// ByteSlice contains a []byte type ByteSlice struct { Bytes []byte } @@ -435,16 +462,18 @@ var _ interfaces.Printable = (*ByteSlice)(nil) var _ interfaces.BinaryMarshallable = (*ByteSlice)(nil) var _ interfaces.BinaryMarshallableAndCopyable = (*ByteSlice)(nil) -func (a *ByteSlice) IsSameAs(b *ByteSlice) bool { - if a == nil || b == nil { - if a == nil && b == nil { +// IsSameAs returns true iff the input ByteSlice is binary identical to this ByteSlice +func (bs *ByteSlice) IsSameAs(b *ByteSlice) bool { + if bs == nil || b == nil { + if bs == nil && b == nil { return true } return false } - return AreBytesEqual(a.Bytes, b.Bytes) + return AreBytesEqual(bs.Bytes, b.Bytes) } +// RandomByteSlice returns a random non empty ByteSlice of length 1 <= len <= 63 func RandomByteSlice() *ByteSlice { bs := new(ByteSlice) x := random.RandNonEmptyByteSlice() @@ -452,6 +481,7 @@ func RandomByteSlice() *ByteSlice { return bs } +// StringToByteSlice converts the input string to a ByteSlice func StringToByteSlice(s string) *ByteSlice { bin, err := DecodeBinary(s) if err != nil { @@ -465,10 +495,12 @@ func StringToByteSlice(s string) *ByteSlice { return bs } +// New returns a new ByteSlice func (bs *ByteSlice) New() interfaces.BinaryMarshallableAndCopyable { return new(ByteSlice) } +// MarshalBinary returns the byte array of the ByteSlice func (bs *ByteSlice) MarshalBinary() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { @@ -478,34 +510,35 @@ func (bs *ByteSlice) MarshalBinary() (rval []byte, err error) { return bs.Bytes[:], nil } +// UnmarshalBinaryData copies the input byte array to the ByteSlice func (bs *ByteSlice) UnmarshalBinaryData(data []byte) (newData []byte, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Error unmarshalling: %v", r) - } - }() bs.Bytes = make([]byte, len(data)) - copy(bs.Bytes[:], data) - return nil, nil + newData, err = UnmarshalBinaryDataOfLength(bs.Bytes[:], data, len(data)) + return } +// UnmarshalBinary unmarshals the input byte array into the ByteSlice func (bs *ByteSlice) UnmarshalBinary(data []byte) (err error) { _, err = bs.UnmarshalBinaryData(data) return } -func (e *ByteSlice) JSONByte() ([]byte, error) { - return EncodeJSON(e) +// JSONByte marshals the ByteSlice into json format +func (bs *ByteSlice) JSONByte() ([]byte, error) { + return EncodeJSON(bs) } -func (e *ByteSlice) JSONString() (string, error) { - return EncodeJSONString(e) +// JSONString marshals the ByteSlice into json format byte string +func (bs *ByteSlice) JSONString() (string, error) { + return EncodeJSONString(bs) } +// String returns a hexidecimal (0-F) string of the ByteSlice func (bs *ByteSlice) String() string { return fmt.Sprintf("%x", bs.Bytes[:]) } +// MarshalText marshals the receiver into text func (bs *ByteSlice) MarshalText() (rval []byte, err error) { defer func(pe *error) { if *pe != nil { From 13f36aa0b0454fd46fef3ba1c9b469fc61be5dcd Mon Sep 17 00:00:00 2001 From: Michael Lam Date: Sat, 3 Aug 2019 11:09:15 -0700 Subject: [PATCH 6/6] Improve Unit Test Coverage --- common/primitives/buffer_test.go | 73 +++++++++++++++++++++++++++++ common/primitives/key_test.go | 44 +++++++++++++++++ common/primitives/merkle_test.go | 15 ++++++ common/primitives/signature_test.go | 10 ++++ common/primitives/util_test.go | 5 ++ 5 files changed, 147 insertions(+) diff --git a/common/primitives/buffer_test.go b/common/primitives/buffer_test.go index 15ba8e0050..e3a72b5b9b 100644 --- a/common/primitives/buffer_test.go +++ b/common/primitives/buffer_test.go @@ -193,6 +193,24 @@ func TestPushPopBytes(t *testing.T) { if AreBytesEqual(b, r) == false { t.Errorf("Received wrong byte slice - %x vs %x", b, r) } + err = buf.Push(b) + if err != nil { + t.Errorf("%v", err) + } + h := make([]byte, len(b)) + err = buf.Pop(h) + if err != nil { + t.Errorf("%v", err) + } + if AreBytesEqual(b, h) == false { + t.Errorf("Received wrong byte slice for push/pop - %x vs %x", b, h) + } + _ = buf.Push(b) // No error, already checked + h2, err := buf.PopLen(len(b)) + if AreBytesEqual(b, h2) == false { + t.Errorf("Received wrong byte slice for PopLen - %x vs %x", b, h2) + } + } } @@ -257,5 +275,60 @@ func TestPushPopBinaryMarshallable(t *testing.T) { if h.IsSameAs(h2) == false { t.Errorf("Received wrong hash - %v vs %v", h, h2) } + err = b.PushIHash(h) + if err != nil { + t.Errorf("%v", err) + } + h3, err := b.PopIHash() + if err != nil { + t.Errorf("%v", err) + } + if h.IsSameAs(h3) == false { + t.Errorf("Received wrong hash - %v vcs %v", h, h3) + } + } +} + +// TestRandomPushPop tests a host of push and pop functions +func TestRandomPushPop(t *testing.T) { + b := NewBuffer(nil) + + r8 := random.RandUInt8() + b.PushUInt8(r8) + b.PushUInt16(32) + rb := random.RandByte() + b.PushByte(rb) + rint := random.RandInt() + b.PushInt(rint) + b.PushBool(true) + b.PushBool(false) + + nr8, err := b.PopUInt8() + if err != nil || nr8 != r8 { + t.Error("Unable to PopUInt8") + } + nr16, err := b.PopUInt16() + if err != nil || nr16 != 32 { + t.Error("Unable to PopUInt16") + } + mybyte, err := b.PeekByte() // Check the byte before popping for a test + if err != nil || mybyte != rb { + t.Error("Unable to PeekByte") + } + nrb, err := b.PopByte() + if err != nil || nrb != rb { + t.Error("Unable to PopByte") + } + nrint, err := b.PopInt() + if err != nil || nrint != rint { + t.Error("Unable to PopInt") + } + nboo, err := b.PopBool() + if err != nil || nboo != true { + t.Error("Unable to PopBool true") + } + nboo, err = b.PopBool() + if err != nil || nboo != false { + t.Error("Unable to PopBool false") } } diff --git a/common/primitives/key_test.go b/common/primitives/key_test.go index 9ce3ca2514..63269ccd33 100644 --- a/common/primitives/key_test.go +++ b/common/primitives/key_test.go @@ -225,4 +225,48 @@ func TestNewPrivateKeyFromHex(t *testing.T) { if AreBytesEqual(pk1.Pub[:], pubKeyBytes[:]) == false { t.Errorf("Public keys are not equal - %x vs %x", pk1.Pub[:], pubKeyBytes[:]) } + + priv2 := pk1.PrivateKeyString() + pub2 := pk1.PublicKeyString() + if priv != priv2 { + t.Error("Could not retrieve private key string") + } + if pub != pub2 { + t.Error("Could not retrieve public key string") + } + + priv3 := RandomPrivateKey() + if priv3.PrivateKeyString() == priv { + // Note: in theory this COULD fail, if you randomly retrieve the same key, but we ignore this small probability + t.Error("Failed difference check") + } +} + +// TestPublicKeyMarshalUnmarshalText tests that a public key can be marshalled/unmarshalled from to/from text properly +func TestPublicKeyMarshalUnmarshalText(t *testing.T) { + pub := PubKeyFromString("my test string") + + t1, err := pub.MarshalText() + if err != nil { + t.Errorf("%v", err) + } + pub2 := new(PublicKey) + err = pub2.UnmarshalText(t1) + if err != nil { + t.Errorf("%v", err) + } + if pub.IsSameAs(pub2) == false { + t.Error("Marshal/Unmarshal Text failed") + } + if pub.IsSameAs(nil) == true { + t.Error("IsSameAs failed on nil input") + } + pub3, err := pub2.Copy() + if err != nil { + t.Errorf("%v", err) + } + if pub.IsSameAs(pub3) == false { + t.Error("Public key copy failed") + } + } diff --git a/common/primitives/merkle_test.go b/common/primitives/merkle_test.go index a231eb00e2..7e7d48a124 100644 --- a/common/primitives/merkle_test.go +++ b/common/primitives/merkle_test.go @@ -84,6 +84,11 @@ func TestBuildMerkleTreeStore(t *testing.T) { } } } + + root := ComputeMerkleRoot(list) + if root.Fixed() != merkles[len(merkles)-1].Fixed() { + t.Errorf("Merkle root discrepancy: %v %v", root, merkles[len(merkles)-1]) + } } func TestBuildMerkleBranch(t *testing.T) { @@ -91,6 +96,7 @@ func TestBuildMerkleBranch(t *testing.T) { list := buildMerkleLeafs(max) tree := buildExpectedMerkleTree(list) branch := BuildMerkleBranch(list, max-1, true) + branch2 := BuildMerkleBranchForHash(list, list[max-1], true) leftIndexes := []int{8, 13, 16, 17} rightIndexes := []int{8, 13, 16, 18} @@ -100,12 +106,21 @@ func TestBuildMerkleBranch(t *testing.T) { if branch[i].Left.IsSameAs(tree[leftIndexes[i]]) == false { t.Errorf("Left node is wrong on index %v", i) } + if branch2[i].Left.IsSameAs(branch[i].Left) == false { + t.Errorf("Left node on branch2 is wrong: %v", i) + } if branch[i].Right.IsSameAs(tree[rightIndexes[i]]) == false { t.Errorf("Right node is wrong on index %v", i) } + if branch2[i].Right.IsSameAs(branch[i].Right) == false { + t.Errorf("Right node on branch2 is wrong on index %v", i) + } if branch[i].Top.IsSameAs(tree[topIndexes[i]]) == false { t.Errorf("Top node is wrong on index %v", i) } + if branch2[i].Top.IsSameAs(branch[i].Top) == false { + t.Errorf("Top node on branch2 is wrong on index %v", i) + } } branch = BuildMerkleBranch(list, 0, true) diff --git a/common/primitives/signature_test.go b/common/primitives/signature_test.go index c46c8a2a6d..e0f4aae9de 100644 --- a/common/primitives/signature_test.go +++ b/common/primitives/signature_test.go @@ -47,6 +47,16 @@ func TestMarshalUnmarshalSignature(t *testing.T) { if len(rest) != 2 { t.Errorf("Invalid rest - %x", rest) } + + if AreBytesEqual(sig.GetPubBytes(), sig.Pub[:]) == false { + t.Errorf("Bad public key detected") + } + if AreBytesEqual(sig.GetSigBytes(), sig.Sig[:]) == false { + t.Errorf("Bad signature") + } + if AreBytesEqual(sig.GetSigBytes(), sig.Bytes()) == false { + t.Errorf("Bad GetBytes") + } } func TestVerifySignature(t *testing.T) { diff --git a/common/primitives/util_test.go b/common/primitives/util_test.go index 512a3239eb..3782393cb4 100644 --- a/common/primitives/util_test.go +++ b/common/primitives/util_test.go @@ -94,6 +94,11 @@ func TestConversions(test *testing.T) { fmt.Println("4", v, err) test.Fail() } + v, err = ConvertFixedPoint("999") + if err != nil || v != "99900000000" { + fmt.Println("5", v, err) + test.Fail() + } } func TestWriteNumber(t *testing.T) {