Algod: support state proofs recoverability

catchup/catchpointService.go

@@ -22,8 +22,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/algorand/go-algorand/stateproof"
-
 	"github.com/algorand/go-deadlock"
 
 	"github.com/algorand/go-algorand/config"
@@ -33,6 +31,7 @@ import (
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/network"
+	"github.com/algorand/go-algorand/stateproof"
 )
 
 const (
@@ -259,6 +258,7 @@ func (cs *CatchpointCatchupService) processStageInactive() (err error) {
 	if err != nil {
 		return cs.abort(fmt.Errorf("processStageInactive failed to set a catchpoint label : %v", err))
 	}
+
 	err = cs.updateStage(ledger.CatchpointCatchupStateLedgerDownload)
 	if err != nil {
 		return cs.abort(fmt.Errorf("processStageInactive failed to update stage : %v", err))

cmd/catchpointdump/database.go

@@ -70,6 +70,10 @@ var databaseCmd = &cobra.Command{
 		if err != nil {
 			reportErrorf("Unable to print key value store : %v", err)
 		}
+		err = printStateProofVerificationContext("./ledger.tracker.sqlite", outFile)
+		if err != nil {
+			reportWarnf("Unable to print state proof verification database : %v", err)
+		}
 	},
 }
 

cmd/catchpointdump/file.go

@@ -47,7 +47,7 @@ import (
 
 var catchpointFile string
 var outFileName string
-var excludedFields *cmdutil.CobraStringSliceValue = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})
+var excludedFields = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})
 
 func init() {
 	fileCmd.Flags().StringVarP(&catchpointFile, "tar", "t", "", "Specify the catchpoint file (either .tar or .tar.gz) to process")
@@ -127,7 +127,6 @@ var fileCmd = &cobra.Command{
 				}
 				defer outFile.Close()
 			}
-
 			err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
 			if err != nil {
 				reportErrorf("Unable to print account database : %v", err)
@@ -136,6 +135,10 @@ var fileCmd = &cobra.Command{
 			if err != nil {
 				reportErrorf("Unable to print key value store : %v", err)
 			}
+			err = printStateProofVerificationContext("./ledger.tracker.sqlite", outFile)
+			if err != nil {
+				reportErrorf("Unable to print state proof verification database : %v", err)
+			}
 		}
 	},
 }
@@ -434,6 +437,38 @@ func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader l
 	})
 }
 
+func printStateProofVerificationContext(databaseName string, outFile *os.File) error {
+	fileWriter := bufio.NewWriterSize(outFile, 1024*1024)
+	defer fileWriter.Flush()
+
+	dbAccessor, err := db.MakeAccessor(databaseName, true, false)
+	if err != nil || dbAccessor.Handle == nil {
+		return err
+	}
+	defer dbAccessor.Close()
+
+	var stateProofVerificationContext []ledgercore.StateProofVerificationContext
+	err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
+		stateProofVerificationContext, err = store.GetAllCatchpointStateProofVerification(ctx, tx)
+		return err
+	})
+
+	if err != nil {
+		return err
+	}
+
+	var printedLines []string
+	for _, ctx := range stateProofVerificationContext {
+		jsonData, err := json.Marshal(ctx)
+		if err != nil {
+			return err
+		}
+		printedLines = append(printedLines, fmt.Sprintf("%d : %s", ctx.LastAttestedRound, string(jsonData)))
+	}
+	_, err = fmt.Fprintf(fileWriter, "State Proof Verification Data:\n"+strings.Join(printedLines, "\n")+"\n")
+	return err
+}
+
 func printKeyValue(writer *bufio.Writer, key, value []byte) {
 	var pretty string
 	ai, rest, err := logic.SplitBoxKey(string(key))

cmd/catchpointdump/net.go

@@ -361,6 +361,10 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
 		if err != nil {
 			return err
 		}
+		err = printStateProofVerificationContext("./ledger.tracker.sqlite", outFile)
+		if err != nil {
+			return err
+		}
 
 	}
 	return nil

cmd/tealdbg/localLedger.go

@@ -285,6 +285,10 @@ func (l *localLedger) BlockHdrCached(basics.Round) (bookkeeping.BlockHeader, err
 	return bookkeeping.BlockHeader{}, nil
 }
 
+func (l *localLedger) GetStateProofVerificationContext(_ basics.Round) (*ledgercore.StateProofVerificationContext, error) {
+	return nil, fmt.Errorf("localLedger: GetStateProofVerificationContext, needed for state proof verification, is not implemented in debugger")
+}
+
 func (l *localLedger) CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error {
 	return nil
 }

config/consensus.go

@@ -483,6 +483,10 @@ type ConsensusParams struct {
 	// the rewardsLevel, but the rewardsLevel has no meaning because the account
 	// has fewer than RewardUnit algos.
 	UnfundedSenders bool
+
+	// StateProofUseTrackerVerification specifies whether the node will use data from state proof verification tracker
+	// in order to verify state proofs.
+	StateProofUseTrackerVerification bool
 }
 
 // PaysetCommitType enumerates possible ways for the block header to commit to
@@ -1250,6 +1254,8 @@ func initConsensusProtocols() {
 
 	vFuture.LogicSigVersion = 9 // When moving this to a release, put a new higher LogicSigVersion here
 
+	vFuture.StateProofUseTrackerVerification = true
+
 	Consensus[protocol.ConsensusFuture] = vFuture
 
 	// vAlphaX versions are an separate series of consensus parameters and versions for alphanet

crypto/stateproof/builder.go

@@ -34,19 +34,28 @@ var (
 	ErrCoinIndexError         = errors.New("could not find corresponding index for a given coin")
 )
 
+// VotersAllocBound should be equal to config.Consensus[protocol.ConsensusCurrentVersion].VotersAllocBound
+const VotersAllocBound = 1024
+
+// BuilderPersistingFields is the set of fields of a Builder that are persisted to disk.
+type BuilderPersistingFields struct {
+	_struct        struct{}             `codec:",omitempty,omitemptyarray"`
+	Data           MessageHash          `codec:"data"`
+	Round          uint64               `codec:"rnd"`
+	Participants   []basics.Participant `codec:"parts,allocbound=VotersAllocBound"`
+	Parttree       *merklearray.Tree    `codec:"parttree"`
+	LnProvenWeight uint64               `codec:"lnprv"`
+	ProvenWeight   uint64               `codec:"prv"`
+	StrengthTarget uint64               `codec:"str"`
+}
+
 // Builder keeps track of signatures on a message and eventually produces
 // a state proof for that message.
 type Builder struct {
-	data           MessageHash
-	round          uint64
-	sigs           []sigslot // Indexed by pos in participants
-	signedWeight   uint64    // Total weight of signatures so far
-	participants   []basics.Participant
-	parttree       *merklearray.Tree
-	lnProvenWeight uint64
-	provenWeight   uint64
-	strengthTarget uint64
-	cachedProof    *StateProof
+	BuilderPersistingFields
+	sigs         []sigslot // Indexed by pos in Participants
+	signedWeight uint64    // Total weight of signatures so far
+	cachedProof  *StateProof
 }
 
 // MakeBuilder constructs an empty builder. After adding enough signatures and signed weight, this builder is used to create a stateproof.
@@ -58,16 +67,19 @@ func MakeBuilder(data MessageHash, round uint64, provenWeight uint64, part []bas
 	}
 
 	b := &Builder{
-		data:           data,
-		round:          round,
-		sigs:           make([]sigslot, npart),
-		signedWeight:   0,
-		participants:   part,
-		parttree:       parttree,
-		lnProvenWeight: lnProvenWt,
-		provenWeight:   provenWeight,
-		strengthTarget: strengthTarget,
-		cachedProof:    nil,
+		BuilderPersistingFields: BuilderPersistingFields{
+			Data:           data,
+			Round:          round,
+			Participants:   part,
+			Parttree:       parttree,
+			LnProvenWeight: lnProvenWt,
+			ProvenWeight:   provenWeight,
+			StrengthTarget: strengthTarget,
+		},
+
+		sigs:         make([]sigslot, npart),
+		signedWeight: 0,
+		cachedProof:  nil,
 	}
 
 	return b, nil
@@ -86,11 +98,11 @@ func (b *Builder) Present(pos uint64) (bool, error) {
 // IsValid verifies that the participant along with the signature can be inserted to the builder.
 // verifySig can be set to false when the signature is already verified (e.g. loaded from the DB)
 func (b *Builder) IsValid(pos uint64, sig *merklesignature.Signature, verifySig bool) error {
-	if pos >= uint64(len(b.participants)) {
-		return fmt.Errorf("%w pos %d >= len(participants) %d", ErrPositionOutOfBound, pos, len(b.participants))
+	if pos >= uint64(len(b.Participants)) {
+		return fmt.Errorf("%w pos %d >= len(participants) %d", ErrPositionOutOfBound, pos, len(b.Participants))
 	}
 
-	p := b.participants[pos]
+	p := b.Participants[pos]
 
 	if p.Weight == 0 {
 		return fmt.Errorf("builder.IsValid: %w: position = %d", ErrPositionWithZeroWeight, pos)
@@ -101,7 +113,7 @@ func (b *Builder) IsValid(pos uint64, sig *merklesignature.Signature, verifySig
 		if err := sig.ValidateSaltVersion(merklesignature.SchemeSaltVersion); err != nil {
 			return err
 		}
-		if err := p.PK.VerifyBytes(b.round, b.data[:], sig); err != nil {
+		if err := p.PK.VerifyBytes(b.Round, b.Data[:], sig); err != nil {
 			return err
 		}
 	}
@@ -118,7 +130,7 @@ func (b *Builder) Add(pos uint64, sig merklesignature.Signature) error {
 		return ErrPositionAlreadyPresent
 	}
 
-	p := b.participants[pos]
+	p := b.Participants[pos]
 
 	// Remember the signature
 	b.sigs[pos].Weight = p.Weight
@@ -130,7 +142,7 @@ func (b *Builder) Add(pos uint64, sig merklesignature.Signature) error {
 
 // Ready returns whether the state proof is ready to be built.
 func (b *Builder) Ready() bool {
-	return b.cachedProof != nil || b.signedWeight > b.provenWeight
+	return b.cachedProof != nil || b.signedWeight > b.ProvenWeight
 }
 
 // SignedWeight returns the total weight of signatures added so far.
@@ -175,7 +187,7 @@ func (b *Builder) Build() (*StateProof, error) {
 	}
 
 	if !b.Ready() {
-		return nil, fmt.Errorf("%w: %d <= %d", ErrSignedWeightLessThanProvenWeight, b.signedWeight, b.provenWeight)
+		return nil, fmt.Errorf("%w: %d <= %d", ErrSignedWeightLessThanProvenWeight, b.signedWeight, b.ProvenWeight)
 	}
 
 	// Commit to the sigs array
@@ -197,17 +209,17 @@ func (b *Builder) Build() (*StateProof, error) {
 		MerkleSignatureSaltVersion: merklesignature.SchemeSaltVersion,
 	}
 
-	nr, err := numReveals(b.signedWeight, b.lnProvenWeight, b.strengthTarget)
+	nr, err := numReveals(b.signedWeight, b.LnProvenWeight, b.StrengthTarget)
 	if err != nil {
 		return nil, err
 	}
 
 	choice := coinChoiceSeed{
-		partCommitment: b.parttree.Root(),
-		lnProvenWeight: b.lnProvenWeight,
+		partCommitment: b.Parttree.Root(),
+		lnProvenWeight: b.LnProvenWeight,
 		sigCommitment:  s.SigCommit,
 		signedWeight:   s.SignedWeight,
-		data:           b.data,
+		data:           b.Data,
 	}
 
 	coinHash := makeCoinGenerator(&choice)
@@ -221,8 +233,8 @@ func (b *Builder) Build() (*StateProof, error) {
 			return nil, err
 		}
 
-		if pos >= uint64(len(b.participants)) {
-			return nil, fmt.Errorf("%w pos %d >= len(participants) %d", ErrPositionOutOfBound, pos, len(b.participants))
+		if pos >= uint64(len(b.Participants)) {
+			return nil, fmt.Errorf("%w pos %d >= len(participants) %d", ErrPositionOutOfBound, pos, len(b.Participants))
 		}
 
 		revealsSequence[j] = pos
@@ -236,7 +248,7 @@ func (b *Builder) Build() (*StateProof, error) {
 		// Generate the reveal for pos
 		s.Reveals[pos] = Reveal{
 			SigSlot: b.sigs[pos].sigslotCommit,
-			Part:    b.participants[pos],
+			Part:    b.Participants[pos],
 		}
 
 		proofPositions = append(proofPositions, pos)
@@ -247,7 +259,7 @@ func (b *Builder) Build() (*StateProof, error) {
 		return nil, err
 	}
 
-	partProofs, err := b.parttree.Prove(proofPositions)
+	partProofs, err := b.Parttree.Prove(proofPositions)
 	if err != nil {
 		return nil, err
 	}
@@ -258,3 +270,8 @@ func (b *Builder) Build() (*StateProof, error) {
 	b.cachedProof = s
 	return s, nil
 }
+
+// AllocSigs should only be used after decoding msgpacked Builder, as the sigs field is not exported and encoded
+func (b *Builder) AllocSigs() {
+	b.sigs = make([]sigslot, len(b.Participants))
+}

crypto/stateproof/builder_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/algorand/falcon"
 
+	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/crypto/merklearray"
 	"github.com/algorand/go-algorand/crypto/merklesignature"
@@ -496,12 +497,12 @@ func TestBuildAndReady(t *testing.T) {
 	_, err = builder.Build()
 	a.ErrorIs(err, ErrSignedWeightLessThanProvenWeight)
 
-	builder.signedWeight = builder.provenWeight
+	builder.signedWeight = builder.ProvenWeight
 	a.False(builder.Ready())
 	_, err = builder.Build()
 	a.ErrorIs(err, ErrSignedWeightLessThanProvenWeight)
 
-	builder.signedWeight = builder.provenWeight + 1
+	builder.signedWeight = builder.ProvenWeight + 1
 	a.True(builder.Ready())
 	_, err = builder.Build()
 	a.NotErrorIs(err, ErrSignedWeightLessThanProvenWeight)
@@ -516,19 +517,19 @@ func TestErrorCases(t *testing.T) {
 	_, err := builder.Present(1)
 	a.ErrorIs(err, ErrPositionOutOfBound)
 
-	builder.participants = make([]basics.Participant, 1, 1)
+	builder.Participants = make([]basics.Participant, 1, 1)
 	builder.sigs = make([]sigslot, 1, 1)
 	err = builder.IsValid(1, &merklesignature.Signature{}, false)
 	a.ErrorIs(err, ErrPositionOutOfBound)
 
 	err = builder.IsValid(0, &merklesignature.Signature{}, false)
 	require.ErrorIs(t, err, ErrPositionWithZeroWeight)
 
-	builder.participants[0].Weight = 1
+	builder.Participants[0].Weight = 1
 	err = builder.IsValid(0, &merklesignature.Signature{}, true)
 	a.ErrorIs(err, merklesignature.ErrKeyLifetimeIsZero)
 
-	builder.participants[0].PK.KeyLifetime = 20
+	builder.Participants[0].PK.KeyLifetime = 20
 	err = builder.IsValid(0, &merklesignature.Signature{}, true)
 	a.ErrorIs(err, merklesignature.ErrSignatureSchemeVerificationFailed)
 
@@ -631,6 +632,13 @@ func TestBuilder_BuildStateProofCache(t *testing.T) {
 	return
 }
 
+// Verifies that the VotersAllocBound constant is equal to the current consensus parameters.
+// It is used for msgpack allocbound (needs to be static)
+func TestBuilder_StateProofTopVoters(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	require.Equal(t, config.Consensus[protocol.ConsensusCurrentVersion].StateProofTopVoters, uint64(VotersAllocBound))
+}
+
 func BenchmarkBuildVerify(b *testing.B) {
 	totalWeight := 1000000
 	npart := 1000