Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tools: let catchpointdump to calculate and print data hashes #5584

Merged
merged 2 commits into from
Jul 20, 2023
Merged

tools: let catchpointdump to calculate and print data hashes #5584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8320a6f
cmd: catchpointdump utility prints hashes
algorandskiy Jul 17, 2023
82dd5ee
improve LedgerService.ServeHTTP logging
algorandskiy Jul 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions catchup/catchpointService_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger"
"github.com/algorand/go-algorand/ledger/ledgercore"
"github.com/algorand/go-algorand/protocol"
"github.com/algorand/go-algorand/test/partitiontest"
)
Expand Down Expand Up @@ -76,6 +77,11 @@ func (m *catchpointCatchupAccessorMock) Ledger() (l ledger.CatchupAccessorClient
return m.l
}

// GetVerifyData returns the balances hash, spver hash and totals used by VerifyCatchpoint
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment appears copy/paste from the real impl, may want to just mention that this is a stub to satisfy the interface.

func (m *catchpointCatchupAccessorMock) GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error) {
return crypto.Digest{}, crypto.Digest{}, ledgercore.AccountTotals{}, nil
}

// TestCatchpointServicePeerRank ensures CatchpointService does not crash when a block fetched
// from the local ledger and not from network when ranking a peer
func TestCatchpointServicePeerRank(t *testing.T) {
Expand Down
15 changes: 15 additions & 0 deletions cmd/catchpointdump/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/algorand/avm-abi/apps"
cmdutil "github.com/algorand/go-algorand/cmd/util"
"github.com/algorand/go-algorand/config"
"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger"
Expand All @@ -48,11 +49,13 @@ import (
var catchpointFile string
var outFileName string
var excludedFields = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})
var printDigests bool

func init() {
fileCmd.Flags().StringVarP(&catchpointFile, "tar", "t", "", "Specify the catchpoint file (either .tar or .tar.gz) to process")
fileCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. tracker.dump.txt )")
fileCmd.Flags().BoolVarP(&loadOnly, "load", "l", false, "Load only, do not dump")
fileCmd.Flags().BoolVarP(&printDigests, "digest", "d", false, "Print balances and spver digests")
fileCmd.Flags().VarP(excludedFields, "exclude-fields", "e", "List of fields to exclude from the dump: ["+excludedFields.AllowedString()+"]")
}

Expand Down Expand Up @@ -206,6 +209,18 @@ func loadCatchpointIntoDatabase(ctx context.Context, catchupAccessor ledger.Catc
header, err := tarReader.Next()
if err != nil {
if err == io.EOF {
if printDigests {
err = catchupAccessor.BuildMerkleTrie(ctx, func(uint64, uint64) {})
if err != nil {
return fileHeader, err
}
var balanceHash, spverHash crypto.Digest
balanceHash, spverHash, _, err = catchupAccessor.GetVerifyData(ctx)
if err != nil {
return fileHeader, err
}
fmt.Printf("accounts digest=%s, spver digest=%s\n\n", balanceHash, spverHash)
}
return fileHeader, nil
}
return fileHeader, err
Expand Down
7 changes: 7 additions & 0 deletions components/mocks/mockCatchpointCatchupAccessor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ package mocks
import (
"context"

"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger"
"github.com/algorand/go-algorand/ledger/ledgercore"
)

// MockCatchpointCatchupAccessor is a dummy CatchpointCatchupAccessor implementation which doesn't do anything.
Expand Down Expand Up @@ -67,6 +69,11 @@ func (m *MockCatchpointCatchupAccessor) GetCatchupBlockRound(ctx context.Context
return basics.Round(0), nil
}

// GetVerifyData returns the balances hash, spver hash and totals used by VerifyCatchpoint
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment appears copy/paste from the real impl, may want to just mention that this is a stub to satisfy the interface.

func (m *MockCatchpointCatchupAccessor) GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error) {
return crypto.Digest{}, crypto.Digest{}, ledgercore.AccountTotals{}, nil
}

// VerifyCatchpoint verifies that the catchpoint is valid by reconstructing the label.
func (m *MockCatchpointCatchupAccessor) VerifyCatchpoint(ctx context.Context, blk *bookkeeping.Block) (err error) {
return nil
Expand Down
69 changes: 40 additions & 29 deletions ledger/catchupaccessor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@
// GetCatchupBlockRound returns the latest block round matching the current catchpoint
GetCatchupBlockRound(ctx context.Context) (round basics.Round, err error)

// GetVerifyData returns the balances hash, spver hash and totals used by VerifyCatchpoint
GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error)

// VerifyCatchpoint verifies that the catchpoint is valid by reconstructing the label.
VerifyCatchpoint(ctx context.Context, blk *bookkeeping.Block) (err error)

Expand Down Expand Up @@ -927,34 +930,9 @@
return basics.Round(iRound), nil
}

// VerifyCatchpoint verifies that the catchpoint is valid by reconstructing the label.
func (c *catchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, blk *bookkeeping.Block) (err error) {
var balancesHash crypto.Digest
func (c *catchpointCatchupAccessorImpl) GetVerifyData(ctx context.Context) (balancesHash crypto.Digest, spverHash crypto.Digest, totals ledgercore.AccountTotals, err error) {
var rawStateProofVerificationContext []ledgercore.StateProofVerificationContext
var blockRound basics.Round
var totals ledgercore.AccountTotals
var catchpointLabel string
var version uint64

catchpointLabel, err = c.catchpointStore.ReadCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel)
if err != nil {
return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err)
}

version, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion)
if err != nil {
return fmt.Errorf("unable to retrieve catchpoint version: %v", err)
}

var iRound uint64
iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound)
if err != nil {
return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBlockRound, err)
}
blockRound = basics.Round(iRound)

start := time.Now()
ledgerVerifycatchpointCount.Inc(nil)
err = c.ledger.trackerDB().Transaction(func(ctx context.Context, tx trackerdb.TransactionScope) (err error) {
ar, err := tx.MakeAccountsReader()
if err != nil {
Expand Down Expand Up @@ -989,6 +967,42 @@

return
})
if err != nil {
return crypto.Digest{}, crypto.Digest{}, ledgercore.AccountTotals{}, err
}

wrappedContext := catchpointStateProofVerificationContext{Data: rawStateProofVerificationContext}
spverHash = crypto.HashObj(wrappedContext)

Check warning on line 975 in ledger/catchupaccessor.go

View check run for this annotation

Codecov / codecov/patch

ledger/catchupaccessor.go#L974-L975 

Added lines #L974 - L975 were not covered by tests

return balancesHash, spverHash, totals, err

Check warning on line 977 in ledger/catchupaccessor.go

View check run for this annotation

Codecov / codecov/patch

ledger/catchupaccessor.go#L977 

Added line #L977 was not covered by tests
}

// VerifyCatchpoint verifies that the catchpoint is valid by reconstructing the label.
func (c *catchpointCatchupAccessorImpl) VerifyCatchpoint(ctx context.Context, blk *bookkeeping.Block) (err error) {
var blockRound basics.Round
var catchpointLabel string
var version uint64

catchpointLabel, err = c.catchpointStore.ReadCatchpointStateString(ctx, trackerdb.CatchpointStateCatchupLabel)
if err != nil {
return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupLabel, err)

Check warning on line 988 in ledger/catchupaccessor.go

View check run for this annotation

Codecov / codecov/patch

ledger/catchupaccessor.go#L988 

Added line #L988 was not covered by tests
}

version, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupVersion)
if err != nil {
return fmt.Errorf("unable to retrieve catchpoint version: %v", err)

Check warning on line 993 in ledger/catchupaccessor.go

View check run for this annotation

Codecov / codecov/patch

ledger/catchupaccessor.go#L993 

Added line #L993 was not covered by tests
}

var iRound uint64
iRound, err = c.catchpointStore.ReadCatchpointStateUint64(ctx, trackerdb.CatchpointStateCatchupBlockRound)
if err != nil {
return fmt.Errorf("unable to read catchpoint catchup state '%s': %v", trackerdb.CatchpointStateCatchupBlockRound, err)

Check warning on line 999 in ledger/catchupaccessor.go

View check run for this annotation

Codecov / codecov/patch

ledger/catchupaccessor.go#L999 

Added line #L999 was not covered by tests
}
blockRound = basics.Round(iRound)

start := time.Now()
ledgerVerifycatchpointCount.Inc(nil)
balancesHash, spVerificationHash, totals, err := c.GetVerifyData(ctx)
ledgerVerifycatchpointMicros.AddMicrosecondsSince(start, nil)
if err != nil {
return err
Expand All @@ -997,9 +1011,6 @@
return fmt.Errorf("block round in block header doesn't match block round in catchpoint: %d != %d", blockRound, blk.Round())
}

wrappedContext := catchpointStateProofVerificationContext{Data: rawStateProofVerificationContext}
spVerificationHash := crypto.HashObj(wrappedContext)

var catchpointLabelMaker ledgercore.CatchpointLabelMaker
blockDigest := blk.Digest()
if version <= CatchpointFileVersionV6 {
Expand Down
25 changes: 16 additions & 9 deletions rpcs/ledgerService.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,21 +117,21 @@
genesisID, hasGenesisID := pathVars["genesisID"]
if hasVersionStr {
if versionStr != "1" {
logging.Base().Debugf("http ledger bad version '%s'", versionStr)
logging.Base().Debugf("LedgerService.ServeHTTP: bad version '%s'", versionStr)
cce marked this conversation as resolved.
Show resolved Hide resolved
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte(fmt.Sprintf("unsupported version '%s'", versionStr)))
return
}
}
if hasGenesisID {
if ls.genesisID != genesisID {
logging.Base().Debugf("http ledger bad genesisID mine=%#v theirs=%#v", ls.genesisID, genesisID)
logging.Base().Debugf("LedgerService.ServeHTTP: bad genesisID mine=%#v theirs=%#v", ls.genesisID, genesisID)
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte(fmt.Sprintf("mismatching genesisID '%s'", genesisID)))
return
}
} else {
logging.Base().Debug("http ledger no genesisID")
logging.Base().Debug("LedgerService.ServeHTTP: no genesisID")
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte("missing genesisID"))
return
Expand All @@ -141,14 +141,14 @@
request.Body = http.MaxBytesReader(response, request.Body, ledgerServerMaxBodyLength)
err := request.ParseForm()
if err != nil {
logging.Base().Debugf("http ledger parse form err : %v", err)
logging.Base().Debugf("LedgerService.ServeHTTP: parse form err : %v", err)

Check warning on line 144 in rpcs/ledgerService.go

View check run for this annotation

Codecov / codecov/patch

rpcs/ledgerService.go#L144 

Added line #L144 was not covered by tests
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte(fmt.Sprintf("unable to parse form body : %v", err)))
return
}
roundStrs, ok := request.Form["r"]
if !ok || len(roundStrs) != 1 {
logging.Base().Debugf("http ledger bad round number form arg '%s'", roundStrs)
logging.Base().Debugf("LedgerService.ServeHTTP: bad round number form arg '%s'", roundStrs)

Check warning on line 151 in rpcs/ledgerService.go

View check run for this annotation

Codecov / codecov/patch

rpcs/ledgerService.go#L151 

Added line #L151 was not covered by tests
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte("invalid round number specified in 'r' form argument"))
return
Expand All @@ -158,13 +158,13 @@
if ok {
if len(versionStrs) == 1 {
if versionStrs[0] != "1" {
logging.Base().Debugf("http ledger bad version '%s'", versionStr)
logging.Base().Debugf("LedgerService.ServeHTTP: bad version '%s'", versionStr)

Check warning on line 161 in rpcs/ledgerService.go

View check run for this annotation

Codecov / codecov/patch

rpcs/ledgerService.go#L161 

Added line #L161 was not covered by tests
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte(fmt.Sprintf("unsupported version specified '%s'", versionStrs[0])))
return
}
} else {
logging.Base().Debugf("http ledger wrong number of v=%d args", len(versionStrs))
logging.Base().Debugf("LedgerService.ServeHTTP: wrong number of v=%d args", len(versionStrs))

Check warning on line 167 in rpcs/ledgerService.go

View check run for this annotation

Codecov / codecov/patch

rpcs/ledgerService.go#L167 

Added line #L167 was not covered by tests
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte(fmt.Sprintf("invalid number of version specified %d", len(versionStrs))))
return
Expand All @@ -173,11 +173,13 @@
}
round, err := strconv.ParseUint(roundStr, 36, 64)
if err != nil {
logging.Base().Debugf("http ledger round parse fail ('%s'): %v", roundStr, err)
logging.Base().Debugf("LedgerService.ServeHTTP: round parse fail ('%s'): %v", roundStr, err)
response.WriteHeader(http.StatusBadRequest)
response.Write([]byte(fmt.Sprintf("specified round number could not be parsed using base 36 : %v", err)))
return
}
logging.Base().Infof("LedgerService.ServeHTTP: serving catchpoint round %d", round)
start := time.Now()
cs, err := ls.ledger.GetCatchpointStream(basics.Round(round))
if err != nil {
switch err.(type) {
Expand All @@ -188,7 +190,7 @@
return
default:
// unexpected error.
logging.Base().Warnf("ServeHTTP : failed to retrieve catchpoint %d %v", round, err)
logging.Base().Warnf("LedgerService.ServeHTTP : failed to retrieve catchpoint %d %v", round, err)
response.WriteHeader(http.StatusInternalServerError)
response.Write([]byte(fmt.Sprintf("catchpoint file for round %d could not be retrieved due to internal error : %v", round, err)))
return
Expand Down Expand Up @@ -221,6 +223,8 @@
if err != nil {
logging.Base().Infof("LedgerService.ServeHTTP : unable to write compressed catchpoint file for round %d, written bytes %d : %v", round, written, err)
}
elapsed := time.Since(start)
logging.Base().Infof("LedgerService.ServeHTTP: served catchpoint round %d in %d sec", round, int(elapsed.Seconds()))

Check warning on line 227 in rpcs/ledgerService.go

View check run for this annotation

Codecov / codecov/patch

rpcs/ledgerService.go#L226-L227 

Added lines #L226 - L227 were not covered by tests
return
}
decompressedGzip, err := gzip.NewReader(cs)
Expand All @@ -234,5 +238,8 @@
written, err := io.Copy(response, decompressedGzip)
if err != nil {
logging.Base().Infof("LedgerService.ServeHTTP : unable to write decompressed catchpoint file for round %d, written bytes %d : %v", round, written, err)
} else {
elapsed := time.Since(start)
logging.Base().Infof("LedgerService.ServeHTTP: served catchpoint round %d in %d sec", round, int(elapsed.Seconds()))

Check warning on line 243 in rpcs/ledgerService.go

View check run for this annotation

Codecov / codecov/patch

rpcs/ledgerService.go#L241-L243 

Added lines #L241 - L243 were not covered by tests
}
}