From 6d72d023e8fdd320472b7311aa09ede32929229d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 13 May 2024 21:07:34 -0700 Subject: [PATCH] EVM-friendly TipSet and ECChain formats This patch implements the changes described in https://github.com/filecoin-project/FIPs/pull/1004. Unfortunately, go-f3 now needs to be aware of the internal structure of `TipSet` objects as it sends them over the wire in one format, but signs another. This effectively reverts #149. Specific changes: 1. On the wire (and in finality certificates), `TipSet` objects are now directly cbor-marshaled within the payload instead of first being marshaled to byte strings. 2. Instead of directly covering the entire tipset list with the payload signature: 1. `TipSet` objects are "marshaled for signing" per the FIP change proposed above. 2. These marshaled tipset objects are stuffed into a merkle tree. 3. The merkle tree root is signed (along with the payload's other fields), again according the proposed changes. fixes #166 --- gen/main.go | 1 + gpbft/chain.go | 138 ++++++++++++++++------ gpbft/chain_test.go | 73 +++--------- gpbft/gen.go | 214 ++++++++++++++++++++++++++++++----- gpbft/gpbft.go | 21 ++-- gpbft/participant_test.go | 14 +-- merkle/merkle.go | 117 +++++++++++++++++++ merkle/merkle_test.go | 39 +++++++ sim/ecchain_gen.go | 4 +- sim/host.go | 2 +- sim/options.go | 2 +- sim/tipset_gen.go | 2 +- test/honest_test.go | 20 ++-- test/multi_instance_test.go | 4 +- test/power_evolution_test.go | 6 +- test/util_test.go | 4 +- 16 files changed, 508 insertions(+), 153 deletions(-) create mode 100644 merkle/merkle.go create mode 100644 merkle/merkle_test.go diff --git a/gen/main.go b/gen/main.go index 49045860..3c6f03f8 100644 --- a/gen/main.go +++ b/gen/main.go @@ -12,6 +12,7 @@ import ( func main() { err := gen.WriteTupleEncodersToFile("../gpbft/gen.go", "gpbft", + gpbft.TipSet{}, gpbft.GMessage{}, gpbft.Payload{}, gpbft.Justification{}, diff --git a/gpbft/chain.go b/gpbft/chain.go index af39d327..c53cc516 100644 --- a/gpbft/chain.go +++ b/gpbft/chain.go @@ -5,16 +5,76 @@ import ( "encoding/binary" "encoding/hex" "errors" + "fmt" "strings" + + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/crypto/blake2b" ) +// TipSetKey is a +type TipSetKey = []byte + +type CID = []byte + +var cidPrefix = []byte{0x01, 0x71, 0xA0, 0xE4, 0x02, 0x20} + +// Hashes the given data and returns a CBOR + blake2b-256 CID. +func MakeCid(data []byte) []byte { + // TODO: Consider just using go-cid? We implicitly depend on it through cbor-gen anyways. + digest := blake2b.Sum256(data) + + out := make([]byte, 0, 38) + out = append(out, cidPrefix...) + out = append(out, digest[:]...) + return out +} + // Opaque type representing a tipset. // This is expected to be: // - a canonical sequence of CIDs of block headers identifying a tipset, // - a commitment to the resulting power table, // - a commitment to additional derived values. // However, GossipPBFT doesn't need to know anything about that structure. -type TipSet = []byte +type TipSet struct { + Epoch int64 + TipSet TipSetKey + PowerTable CID + Commitments [32]byte +} + +func (ts *TipSet) IsZero() bool { + return len(ts.TipSet) == 0 +} + +func (ts *TipSet) Equal(b *TipSet) bool { + return ts.Epoch == b.Epoch && + bytes.Equal(ts.TipSet, b.TipSet) && + bytes.Equal(ts.PowerTable, b.PowerTable) && + ts.Commitments == b.Commitments +} + +func (ts *TipSet) MarshalForSigning() []byte { + var buf bytes.Buffer + cbg.WriteByteArray(&buf, ts.TipSet) + tsCid := MakeCid(buf.Bytes()) + buf.Reset() + buf.Grow(len(tsCid) + len(ts.PowerTable) + 32 + 8) + // epoch || commitments || tipset || powertable + _ = binary.Write(&buf, binary.BigEndian, ts.Epoch) + _, _ = buf.Write(ts.Commitments[:]) + _, _ = buf.Write(tsCid) + _, _ = buf.Write(ts.PowerTable) + return buf.Bytes() +} + +func (ts *TipSet) String() string { + if ts == nil { + return "" + } + + return fmt.Sprintf("%d@%s", ts.Epoch, hex.EncodeToString(ts.TipSet)) +} // A chain of tipsets comprising a base (the last finalised tipset from which the chain extends). // and (possibly empty) suffix. @@ -45,8 +105,8 @@ func (c ECChain) IsZero() bool { } // Returns the base tipset. -func (c ECChain) Base() TipSet { - return c[0] +func (c ECChain) Base() *TipSet { + return &c[0] } // Returns the suffix of the chain after the base. @@ -61,8 +121,8 @@ func (c ECChain) Suffix() []TipSet { // Returns the last tipset in the chain. // This could be the base tipset if there is no suffix. // This will panic on a zero value. -func (c ECChain) Head() TipSet { - return c[len(c)-1] +func (c ECChain) Head() *TipSet { + return &c[len(c)-1] } // Returns a new chain with the same base and no suffix. @@ -71,8 +131,15 @@ func (c ECChain) BaseChain() ECChain { return ECChain{c[0]} } -func (c ECChain) Extend(tip ...TipSet) ECChain { - return append(c, tip...) +func (c ECChain) Extend(tips ...TipSetKey) ECChain { + offset := c.Head().Epoch + 1 + for i, tip := range tips { + c = append(c, TipSet{ + Epoch: offset + int64(i), + TipSet: tip, + }) + } + return c } // Returns a chain with suffix (after the base) truncated to a maximum length. @@ -88,9 +155,7 @@ func (c ECChain) Eq(other ECChain) bool { return false } for i := range c { - if !bytes.Equal(c[i], other[i]) { - return false - } + c[i].Equal(&other[i]) } return true } @@ -101,16 +166,13 @@ func (c ECChain) SameBase(other ECChain) bool { if c.IsZero() || other.IsZero() { return false } - return bytes.Equal(c.Base(), other.Base()) + return c.Base().Equal(other.Base()) } // Check whether a chain has a specific base tipset. // Always false for a zero value. -func (c ECChain) HasBase(t TipSet) bool { - if c.IsZero() || len(t) == 0 { - return false - } - return bytes.Equal(c[0], t) +func (c ECChain) HasBase(t *TipSet) bool { + return !t.IsZero() && !c.IsZero() && c.Base().Equal(t) } // Checks whether a chain has some prefix (including the base). @@ -123,7 +185,7 @@ func (c ECChain) HasPrefix(other ECChain) bool { return false } for i := range other { - if !bytes.Equal(c[i], other[i]) { + if !c[i].Equal(&other[i]) { return false } } @@ -131,13 +193,13 @@ func (c ECChain) HasPrefix(other ECChain) bool { } // Checks whether a chain has some tipset (including as its base). -func (c ECChain) HasTipset(t TipSet) bool { - if len(t) == 0 { +func (c ECChain) HasTipset(t *TipSet) bool { + if t.IsZero() { // Chain can never contain zero-valued TipSet. return false } - for _, t2 := range c { - if bytes.Equal(t, t2) { + for i := range c { + if c[i].Equal(t) { return true } } @@ -147,7 +209,8 @@ func (c ECChain) HasTipset(t TipSet) bool { // Validates a chain value, returning an error if it finds any issues. // A chain is valid if it meets the following criteria: // 1) All contained tipsets are non-empty. -// 2) The chain is not longer than CHAIN_MAX_LEN. +// 2) All epochs are >= 0 and increasing. +// 3) The chain is not longer than CHAIN_MAX_LEN. // An entirely zero-valued chain itself is deemed valid. See ECChain.IsZero. func (c ECChain) Validate() error { if c.IsZero() { @@ -156,10 +219,16 @@ func (c ECChain) Validate() error { if len(c) > CHAIN_MAX_LEN { return errors.New("chain too long") } - for _, tipSet := range c { - if len(tipSet) == 0 { + var lastEpoch int64 = -1 + for i := range c { + ts := &c[i] + if ts.IsZero() { return errors.New("chain cannot contain zero-valued tip sets") } + if ts.Epoch <= lastEpoch { + return errors.New("chain must have increasing epochs") + } + lastEpoch = ts.Epoch } return nil } @@ -167,16 +236,19 @@ func (c ECChain) Validate() error { // Returns an identifier for the chain suitable for use as a map key. // This must completely determine the sequence of tipsets in the chain. func (c ECChain) Key() ChainKey { - var ln int - for _, t := range c { - ln += 4 // for length - ln += len(t) // for data + ln := len(c) * (8 + 32 + 4) // epoch + commitement + ts length + for i := range c { + ln += len(c[i].TipSet) + len(c[i].PowerTable) } var buf bytes.Buffer buf.Grow(ln) - for _, t := range c { - _ = binary.Write(&buf, binary.BigEndian, uint32(len(t))) - buf.Write(t) + for i := range c { + ts := &c[i] + _ = binary.Write(&buf, binary.BigEndian, ts.Epoch) + _, _ = buf.Write(ts.Commitments[:]) + _ = binary.Write(&buf, binary.BigEndian, uint32(len(ts.TipSet))) + buf.Write(ts.TipSet) + _, _ = buf.Write(ts.PowerTable) } return ChainKey(buf.String()) } @@ -184,8 +256,8 @@ func (c ECChain) Key() ChainKey { func (c ECChain) String() string { var b strings.Builder b.WriteString("[") - for i, t := range c { - b.WriteString(hex.EncodeToString(t)) + for i := range c { + b.WriteString(c[i].String()) if i < len(c)-1 { b.WriteString(", ") } diff --git a/gpbft/chain_test.go b/gpbft/chain_test.go index 42630b63..7739f62b 100644 --- a/gpbft/chain_test.go +++ b/gpbft/chain_test.go @@ -7,56 +7,17 @@ import ( "github.com/stretchr/testify/require" ) -func TestTipSet(t *testing.T) { - t.Parallel() - tests := []struct { - name string - subject gpbft.TipSet - wantZero bool - wantString string - }{ - { - name: "zero-value struct is zero", - wantZero: true, - wantString: "", - }, - { - name: "ZeroTipSet is zero", - subject: []byte{}, - wantZero: true, - wantString: "", - }, - { - name: "NewTipSet with zero values is zero", - subject: nil, - wantZero: true, - wantString: "", - }, - { - name: "Non-zero is not zero", - subject: gpbft.TipSet("fish"), - wantString: "fish", - }, - } - for _, test := range tests { - test := test - t.Run(test.name, func(t *testing.T) { - require.Equal(t, test.wantZero, len(test.subject) == 0) - require.Equal(t, test.wantString, string(test.subject)) - }) - } -} - func TestECChain(t *testing.T) { t.Parallel() - zeroTipSet := []byte{} + zeroTipSet := gpbft.TipSet{} + oneTipSet := gpbft.TipSet{Epoch: 0, TipSet: []byte{1}} t.Run("zero-value is zero", func(t *testing.T) { var subject gpbft.ECChain require.True(t, subject.IsZero()) - require.False(t, subject.HasBase(zeroTipSet)) + require.False(t, subject.HasBase(&zeroTipSet)) require.False(t, subject.HasPrefix(subject)) - require.False(t, subject.HasTipset(zeroTipSet)) + require.False(t, subject.HasTipset(&zeroTipSet)) require.False(t, subject.SameBase(subject)) require.True(t, subject.Eq(subject)) require.True(t, subject.Eq(*new(gpbft.ECChain))) @@ -72,31 +33,31 @@ func TestECChain(t *testing.T) { require.Nil(t, subject) }) t.Run("extended chain is as expected", func(t *testing.T) { - wantBase := []byte("fish") + wantBase := gpbft.TipSet{Epoch: 0, TipSet: []byte("fish")} subject, err := gpbft.NewChain(wantBase) require.NoError(t, err) require.Len(t, subject, 1) - require.Equal(t, wantBase, subject.Base()) - require.Equal(t, wantBase, subject.Head()) + require.Equal(t, &wantBase, subject.Base()) + require.Equal(t, &wantBase, subject.Head()) require.NoError(t, subject.Validate()) - wantNext := []byte("lobster") - subjectExtended := subject.Extend(wantNext) + wantNext := gpbft.TipSet{Epoch: 1, TipSet: []byte("lobster")} + subjectExtended := subject.Extend(wantNext.TipSet) require.Len(t, subjectExtended, 2) require.NoError(t, subjectExtended.Validate()) - require.Equal(t, wantBase, subjectExtended.Base()) + require.Equal(t, &wantBase, subjectExtended.Base()) require.Equal(t, []gpbft.TipSet{wantNext}, subjectExtended.Suffix()) - require.Equal(t, wantNext, subjectExtended.Head()) - require.Equal(t, wantNext, subjectExtended.Prefix(1).Head()) - require.True(t, subjectExtended.HasTipset(wantBase)) + require.Equal(t, &wantNext, subjectExtended.Head()) + require.Equal(t, &wantNext, subjectExtended.Prefix(1).Head()) + require.True(t, subjectExtended.HasTipset(&wantBase)) require.False(t, subject.HasPrefix(subjectExtended)) require.True(t, subjectExtended.HasPrefix(subject)) - require.False(t, subject.Extend(wantBase).HasPrefix(subjectExtended.Extend(wantNext))) + require.False(t, subject.Extend(wantBase.TipSet).HasPrefix(subjectExtended.Extend(wantNext.TipSet))) }) t.Run("SameBase is false when either chain is zero", func(t *testing.T) { var zeroChain gpbft.ECChain - nonZeroChain, err := gpbft.NewChain([]byte{1}) + nonZeroChain, err := gpbft.NewChain(oneTipSet) require.NoError(t, err) require.False(t, nonZeroChain.SameBase(zeroChain)) require.False(t, zeroChain.SameBase(nonZeroChain)) @@ -104,7 +65,7 @@ func TestECChain(t *testing.T) { }) t.Run("HasPrefix is false when either chain is zero", func(t *testing.T) { var zeroChain gpbft.ECChain - nonZeroChain, err := gpbft.NewChain([]byte{1}) + nonZeroChain, err := gpbft.NewChain(oneTipSet) require.NoError(t, err) require.False(t, nonZeroChain.HasPrefix(zeroChain)) require.False(t, zeroChain.HasPrefix(nonZeroChain)) @@ -115,7 +76,7 @@ func TestECChain(t *testing.T) { require.NoError(t, zeroChain.Validate()) }) t.Run("ordered chain with zero-valued base is invalid", func(t *testing.T) { - subject := gpbft.ECChain{zeroTipSet, []byte{1}} + subject := gpbft.ECChain{zeroTipSet, oneTipSet} require.Error(t, subject.Validate()) }) } diff --git a/gpbft/gen.go b/gpbft/gen.go index 0d52e192..127e980b 100644 --- a/gpbft/gen.go +++ b/gpbft/gen.go @@ -18,6 +18,188 @@ var _ = cid.Undef var _ = math.E var _ = sort.Sort +var lengthBufTipSet = []byte{132} + +func (t *TipSet) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufTipSet); err != nil { + return err + } + + // t.Epoch (int64) (int64) + if t.Epoch >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil { + return err + } + } + + // t.TipSet ([]uint8) (slice) + if len(t.TipSet) > 2097152 { + return xerrors.Errorf("Byte array in field t.TipSet was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.TipSet))); err != nil { + return err + } + + if _, err := cw.Write(t.TipSet); err != nil { + return err + } + + // t.PowerTable ([]uint8) (slice) + if len(t.PowerTable) > 2097152 { + return xerrors.Errorf("Byte array in field t.PowerTable was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.PowerTable))); err != nil { + return err + } + + if _, err := cw.Write(t.PowerTable); err != nil { + return err + } + + // t.Commitments ([32]uint8) (array) + if len(t.Commitments) > 2097152 { + return xerrors.Errorf("Byte array in field t.Commitments was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Commitments))); err != nil { + return err + } + + if _, err := cw.Write(t.Commitments[:]); err != nil { + return err + } + return nil +} + +func (t *TipSet) UnmarshalCBOR(r io.Reader) (err error) { + *t = TipSet{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 4 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Epoch (int64) (int64) + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Epoch = int64(extraI) + } + // t.TipSet ([]uint8) (slice) + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.TipSet: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.TipSet = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.TipSet); err != nil { + return err + } + + // t.PowerTable ([]uint8) (slice) + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.PowerTable: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.PowerTable = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.PowerTable); err != nil { + return err + } + + // t.Commitments ([32]uint8) (array) + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Commitments: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + if extra != 32 { + return fmt.Errorf("expected array to have 32 elements") + } + + t.Commitments = [32]uint8{} + if _, err := io.ReadFull(cr, t.Commitments[:]); err != nil { + return err + } + return nil +} + var lengthBufGMessage = []byte{133} func (t *GMessage) MarshalCBOR(w io.Writer) error { @@ -228,15 +410,7 @@ func (t *Payload) MarshalCBOR(w io.Writer) error { return err } for _, v := range t.Value { - if len(v) > 2097152 { - return xerrors.Errorf("Byte array in field v was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(v))); err != nil { - return err - } - - if _, err := cw.Write(v); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } @@ -324,7 +498,7 @@ func (t *Payload) UnmarshalCBOR(r io.Reader) (err error) { } if extra > 0 { - t.Value = make([][]uint8, extra) + t.Value = make([]TipSet, extra) } for i := 0; i < int(extra); i++ { @@ -336,24 +510,12 @@ func (t *Payload) UnmarshalCBOR(r io.Reader) (err error) { _ = extra _ = err - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - - if extra > 2097152 { - return fmt.Errorf("t.Value[i]: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } + { - if extra > 0 { - t.Value[i] = make([]uint8, extra) - } + if err := t.Value[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Value[i]: %w", err) + } - if _, err := io.ReadFull(cr, t.Value[i]); err != nil { - return err } } diff --git a/gpbft/gpbft.go b/gpbft/gpbft.go index aed16714..f1ba3794 100644 --- a/gpbft/gpbft.go +++ b/gpbft/gpbft.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-bitfield" rlepluslazy "github.com/filecoin-project/go-bitfield/rle" + "github.com/filecoin-project/go-f3/merkle" "golang.org/x/xerrors" ) @@ -90,26 +91,30 @@ type Payload struct { Value ECChain } -func (p Payload) Eq(other *Payload) bool { +func (p *Payload) Eq(other *Payload) bool { return p.Instance == other.Instance && p.Round == other.Round && p.Step == other.Step && p.Value.Eq(other.Value) } -func (p Payload) MarshalForSigning(nn NetworkName) []byte { +func (p *Payload) MarshalForSigning(nn NetworkName) []byte { + values := make([][]byte, len(p.Value)) + for i := range p.Value { + values[i] = p.Value[i].MarshalForSigning() + } + root := merkle.Tree(values) + var buf bytes.Buffer buf.WriteString(DOMAIN_SEPARATION_TAG) buf.WriteString(":") buf.WriteString(string(nn)) buf.WriteString(":") - _ = binary.Write(&buf, binary.BigEndian, p.Instance) - _ = binary.Write(&buf, binary.BigEndian, p.Round) + _ = binary.Write(&buf, binary.BigEndian, p.Step) - for _, t := range p.Value { - _ = binary.Write(&buf, binary.BigEndian, uint32(len(t))) - buf.Write(t) - } + _ = binary.Write(&buf, binary.BigEndian, p.Round) + _ = binary.Write(&buf, binary.BigEndian, p.Instance) + _, _ = buf.Write(root[:]) return buf.Bytes() } diff --git a/gpbft/participant_test.go b/gpbft/participant_test.go index a45cf739..0ae6e3b8 100644 --- a/gpbft/participant_test.go +++ b/gpbft/participant_test.go @@ -39,7 +39,7 @@ type participantTestSubject struct { func newParticipantTestSubject(t *testing.T, seed int64, instance uint64) *participantTestSubject { // Generate some canonical chain. - canonicalChain, err := gpbft.NewChain([]byte("genesis")) + canonicalChain, err := gpbft.NewChain(gpbft.TipSet{Epoch: 0, TipSet: []byte("genesis")}) require.NoError(t, err) const ( @@ -258,7 +258,7 @@ func TestParticipant(t *testing.T) { }) t.Run("on invalid canonical chain", func(t *testing.T) { subject := newParticipantTestSubject(t, seed, 0) - invalidChain := gpbft.ECChain{nil} + invalidChain := gpbft.ECChain{gpbft.TipSet{}} subject.host.On("GetCanonicalChain").Return(invalidChain, *subject.powerTable, subject.beacon) require.ErrorContains(t, subject.Start(), "invalid canonical chain") subject.assertHostExpectations() @@ -291,7 +291,7 @@ func TestParticipant(t *testing.T) { Sender: somePowerEntry.ID, Vote: gpbft.Payload{ Instance: initialInstance, - Value: gpbft.ECChain{nil}, + Value: gpbft.ECChain{gpbft.TipSet{}}, }, }, false }, @@ -404,7 +404,7 @@ func TestParticipant_ValidateMessage(t *testing.T) { Sender: somePowerEntry.ID, Vote: gpbft.Payload{ Instance: initialInstanceNumber, - Value: gpbft.ECChain{[]byte("fish")}, + Value: gpbft.ECChain{gpbft.TipSet{Epoch: 0, TipSet: []byte("fish")}}, }, } }, @@ -417,7 +417,7 @@ func TestParticipant_ValidateMessage(t *testing.T) { Sender: somePowerEntry.ID, Vote: gpbft.Payload{ Instance: initialInstanceNumber, - Value: gpbft.ECChain{subject.canonicalChain.Base(), nil}, + Value: gpbft.ECChain{*subject.canonicalChain.Base(), gpbft.TipSet{}}, }, } }, @@ -511,7 +511,7 @@ func TestParticipant_ValidateMessage(t *testing.T) { Instance: initialInstanceNumber, Step: gpbft.CONVERGE_PHASE, Round: 42, - Value: gpbft.ECChain{subject.canonicalChain.Base(), nil}, + Value: gpbft.ECChain{*subject.canonicalChain.Base(), gpbft.TipSet{}}, }, } }, @@ -808,7 +808,7 @@ func TestParticipant_ValidateMessage(t *testing.T) { Justification: &gpbft.Justification{ Vote: gpbft.Payload{ Instance: initialInstanceNumber, - Value: gpbft.ECChain{subject.canonicalChain.Base(), nil}, + Value: gpbft.ECChain{*subject.canonicalChain.Base(), gpbft.TipSet{}}, }, }, } diff --git a/merkle/merkle.go b/merkle/merkle.go new file mode 100644 index 00000000..c3bd3315 --- /dev/null +++ b/merkle/merkle.go @@ -0,0 +1,117 @@ +package merkle + +import ( + "math" + "math/bits" + + "golang.org/x/crypto/sha3" +) + +// Digest is a 32-byte hash digest. +type Digest = [32]byte + +// TreeWithProofs returns a the root of the merkle-tree of the given values, along with merkle-proofs for +// each leaf. +func TreeWithProofs(values [][]byte) (Digest, [][]Digest) { + depth := depth(values) + proofs := make([][]Digest, len(values)) + for i := range proofs { + proofs[i] = make([]Digest, 0, depth) + } + return buildTree(depth, values, proofs), proofs +} + +// Tree returns a the root of the merkle-tree of the given values. +func Tree(values [][]byte) Digest { + return buildTree(bits.Len(uint(len(values))-1), values, nil) +} + +// VerifyProof verifies that the given value maps to the given index in the merkle-tree with the +// given root. It returns "more" if the value is not the last value in the merkle-tree. +func VerifyProof(root Digest, index int, value []byte, proof []Digest) (valid bool, more bool) { + // We only allow int32 items, assert that. + if index >= math.MaxInt32 || len(proof) >= 32 { + return false, false + } + + // Make sure the index is in-range for the proof. + if index > (1< 0 { + leftProofs = proofs[:split] + rightProofs = proofs[split:] + } + + leftHash := buildTree(depth-1, values[:split], leftProofs) + rightHash := buildTree(depth-1, values[split:], rightProofs) + + for i, proof := range leftProofs { + leftProofs[i] = append(proof, rightHash) + } + for i, proof := range rightProofs { + rightProofs[i] = append(proof, leftHash) + } + + return internalHash(leftHash, rightHash) +} diff --git a/merkle/merkle_test.go b/merkle/merkle_test.go new file mode 100644 index 00000000..e2137b80 --- /dev/null +++ b/merkle/merkle_test.go @@ -0,0 +1,39 @@ +package merkle + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHashTree(t *testing.T) { + for i := 1; i < 256; i++ { + t.Run(fmt.Sprintf("Length/%d", i), func(t *testing.T) { + test := make([][]byte, i) + for j := range test { + test[j] = []byte{byte(j)} + + } + root, paths := TreeWithProofs(test) + root2 := Tree(test) + require.Equal(t, root, root2) + require.Equal(t, len(test), len(paths)) + assert.Equal(t, len(paths[0]), depth(test)) + + for i, path := range paths { + valid, more := VerifyProof(root, i, test[i], path) + assert.True(t, valid, "proof was not valid for index %d", i) + assert.Equal(t, i < len(paths)-1, more, "incorrect value for 'more' for index %d", i) + } + }) + } +} + +func TestHashZero(t *testing.T) { + test := [][]byte{} + root, paths := TreeWithProofs(test) + assert.Empty(t, paths) + assert.Equal(t, root, Digest{}) +} diff --git a/sim/ecchain_gen.go b/sim/ecchain_gen.go index 22a318dd..971c01a9 100644 --- a/sim/ecchain_gen.go +++ b/sim/ecchain_gen.go @@ -141,9 +141,7 @@ func NewAppendingECChainGenerator(g ...ECChainGenerator) *AggregateECChainGenera func (u *AggregateECChainGenerator) GenerateECChain(instance uint64, base gpbft.TipSet, participant gpbft.ActorID) gpbft.ECChain { chain := gpbft.ECChain{base} for _, generator := range u.generators { - chain = chain.Extend( - generator.GenerateECChain(instance, chain.Head(), participant). - Suffix()...) + chain = append(chain, generator.GenerateECChain(instance, *chain.Head(), participant).Suffix()...) } return chain } diff --git a/sim/host.go b/sim/host.go index 36d920e7..600da5d2 100644 --- a/sim/host.go +++ b/sim/host.go @@ -48,7 +48,7 @@ func newHost(id gpbft.ActorID, sim *Simulation, ecg ECChainGenerator, spg Storag func (v *simHost) GetCanonicalChain() (gpbft.ECChain, gpbft.PowerTable, []byte) { i := v.sim.ec.GetInstance(v.instance) // Use the head of latest agreement chain as the base of next. - chain := v.ecg.GenerateECChain(v.instance, v.ecChain.Head(), v.id) + chain := v.ecg.GenerateECChain(v.instance, *v.ecChain.Head(), v.id) return chain, *i.PowerTable, i.Beacon } diff --git a/sim/options.go b/sim/options.go index 4303769a..525b093d 100644 --- a/sim/options.go +++ b/sim/options.go @@ -22,7 +22,7 @@ var ( func init() { var err error - defaultBaseChain, err = gpbft.NewChain([]byte(("genesis"))) + defaultBaseChain, err = gpbft.NewChain(gpbft.TipSet{Epoch: 0, TipSet: []byte("genesis")}) if err != nil { panic("failed to instantiate default simulation base chain") } diff --git a/sim/tipset_gen.go b/sim/tipset_gen.go index 41751c98..377cb5c4 100644 --- a/sim/tipset_gen.go +++ b/sim/tipset_gen.go @@ -15,7 +15,7 @@ func NewTipSetGenerator(seed uint64) *TipSetGenerator { return &TipSetGenerator{xorshiftState: seed} } -func (c *TipSetGenerator) Sample() gpbft.TipSet { +func (c *TipSetGenerator) Sample() gpbft.TipSetKey { b := make([]byte, 8) for i := range b { b[i] = alphanum[c.nextN(len(alphanum))] diff --git a/test/honest_test.go b/test/honest_test.go index ad4a934b..3785c614 100644 --- a/test/honest_test.go +++ b/test/honest_test.go @@ -60,7 +60,7 @@ func TestHonest_ChainAgreement(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) - requireConsensusAtFirstInstance(t, sm, targetChain.Head()) + requireConsensusAtFirstInstance(t, sm, *targetChain.Head()) }) } } @@ -122,7 +122,7 @@ func TestSync_AgreementWithRepetition(t *testing.T) { require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) // Synchronous, agreeing groups always decide the candidate. - requireConsensusAtFirstInstance(t, sm, someChain.Head()) + requireConsensusAtFirstInstance(t, sm, *someChain.Head()) }) } @@ -149,7 +149,7 @@ func TestAsyncAgreement(t *testing.T) { )...) require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) - requireConsensusAtFirstInstance(t, sm, baseChain.Head(), someChain.Head()) + requireConsensusAtFirstInstance(t, sm, *baseChain.Head(), *someChain.Head()) }) }) } @@ -172,7 +172,7 @@ func TestSyncHalves(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) // Groups split 50/50 always decide the base. - requireConsensusAtFirstInstance(t, sm, baseChain.Head()) + requireConsensusAtFirstInstance(t, sm, *baseChain.Head()) }) } @@ -197,7 +197,7 @@ func TestSyncHalvesBLS(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) // Groups split 50/50 always decide the base. - requireConsensusAtFirstInstance(t, sm, baseChain.Head()) + requireConsensusAtFirstInstance(t, sm, *baseChain.Head()) }) } @@ -223,7 +223,7 @@ func TestAsyncHalves(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) // Groups split 50/50 always decide the base. - requireConsensusAtFirstInstance(t, sm, baseChain.Head()) + requireConsensusAtFirstInstance(t, sm, *baseChain.Head()) }) }) } @@ -248,7 +248,7 @@ func TestRequireStrongQuorumToProgress(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) // Must decide base. - requireConsensusAtFirstInstance(t, sm, baseChain.Head()) + requireConsensusAtFirstInstance(t, sm, *baseChain.Head()) }) } @@ -272,7 +272,7 @@ func TestHonest_FixedLongestCommonPrefix(t *testing.T) { require.NoError(t, err) require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe()) // Must decide ab, the longest common prefix. - requireConsensusAtFirstInstance(t, sm, commonPrefix.Head()) + requireConsensusAtFirstInstance(t, sm, *commonPrefix.Head()) } // TestHonest_MajorityCommonPrefix tests that in a network of honest participants, where there is a majority @@ -302,7 +302,7 @@ func TestHonest_MajorityCommonPrefix(t *testing.T) { for i := 0; i < instanceCount; i++ { ii := uint64(i) instance := sm.GetInstance(ii) - commonPrefix := majorityCommonPrefixGenerator.GenerateECChain(ii, instance.BaseChain.Base(), 0) - requireConsensusAtInstance(t, sm, ii, commonPrefix.Head()) + commonPrefix := majorityCommonPrefixGenerator.GenerateECChain(ii, *instance.BaseChain.Base(), 0) + requireConsensusAtInstance(t, sm, ii, *commonPrefix.Head()) } } diff --git a/test/multi_instance_test.go b/test/multi_instance_test.go index aafafadd..bb389f4e 100644 --- a/test/multi_instance_test.go +++ b/test/multi_instance_test.go @@ -20,7 +20,7 @@ func TestMultiSingleton(t *testing.T) { instance := sm.GetInstance(instanceCount) require.NotNil(t, instance) expected := instance.BaseChain - requireConsensusAtInstance(t, sm, instanceCount-1, expected.Head()) + requireConsensusAtInstance(t, sm, instanceCount-1, *expected.Head()) } func TestMultiSyncPair(t *testing.T) { @@ -32,7 +32,7 @@ func TestMultiSyncPair(t *testing.T) { instance := sm.GetInstance(instanceCount) require.NotNil(t, instance) expected := instance.BaseChain - requireConsensusAtInstance(t, sm, instanceCount-1, expected.Head()) + requireConsensusAtInstance(t, sm, instanceCount-1, *expected.Head()) } func TestMultiASyncPair(t *testing.T) { diff --git a/test/power_evolution_test.go b/test/power_evolution_test.go index 3c669926..6ae8a5b0 100644 --- a/test/power_evolution_test.go +++ b/test/power_evolution_test.go @@ -68,7 +68,7 @@ func TestStoragePower_IncreaseMidSimulation(t *testing.T) { // Assert that the chains agreed upon belong to group 1 before instance 4 and // to group 2 after that. - base := baseChain.Head() + base := *baseChain.Head() for i := uint64(0); i < instanceCount-1; i++ { instance := sm.GetInstance(i + 1) require.NotNil(t, instance, "instance %d", i) @@ -90,7 +90,7 @@ func TestStoragePower_IncreaseMidSimulation(t *testing.T) { // Assert the consensus is reached on the chain with most power. requireConsensusAtInstance(t, sm, i, chainBackedByMostPower...) - base = instance.BaseChain.Head() + base = *instance.BaseChain.Head() } }) }) @@ -161,7 +161,7 @@ func TestStoragePower_DecreaseRevertsToBase(t *testing.T) { // Assert that the head tipset of all decisions made by participants is the base // of instance's base-chain. - requireConsensusAtInstance(t, sm, i, instance.BaseChain.Base()) + requireConsensusAtInstance(t, sm, i, *instance.BaseChain.Base()) } }) }) diff --git a/test/util_test.go b/test/util_test.go index c986b79a..464b84c5 100644 --- a/test/util_test.go +++ b/test/util_test.go @@ -39,7 +39,7 @@ func requireConsensusAtInstance(t *testing.T, sm *sim.Simulation, instance uint6 require.NotNil(t, inst, "no such instance") decision := inst.GetDecision(pid) require.NotNil(t, decision, "no decision for participant %d in instance %d", pid, instance) - require.Contains(t, expectAnyOf, decision.Head(), "consensus not reached: participant %d decided %s in instance %d, expected any of %s", + require.Contains(t, expectAnyOf, *decision.Head(), "consensus not reached: participant %d decided %s in instance %d, expected any of %s", pid, decision.Head(), instance, expectAnyOf) } } @@ -47,7 +47,7 @@ func requireConsensusAtInstance(t *testing.T, sm *sim.Simulation, instance uint6 func generateECChain(t *testing.T, tsg *sim.TipSetGenerator) gpbft.ECChain { t.Helper() // TODO: add stochastic chain generation. - chain, err := gpbft.NewChain(tsg.Sample()) + chain, err := gpbft.NewChain(gpbft.TipSet{Epoch: 0, TipSet: tsg.Sample()}) require.NoError(t, err) return chain }