From 90d347a791d2c6df761603edb8ad7ca895ba8207 Mon Sep 17 00:00:00 2001 From: alainjr10 Date: Fri, 5 Jul 2024 13:40:16 +0100 Subject: [PATCH] fix: rm redundant storing of roots --- blockchain/indexers/cfindex.go | 2 - blockchain/indexers/flatutreexoproofindex.go | 19 +- blockchain/indexers/utreexoproofindex.go | 26 +- server.go | 405 +++++++++++++------ 4 files changed, 311 insertions(+), 141 deletions(-) diff --git a/blockchain/indexers/cfindex.go b/blockchain/indexers/cfindex.go index 044a1734..72ea66cf 100644 --- a/blockchain/indexers/cfindex.go +++ b/blockchain/indexers/cfindex.go @@ -34,7 +34,6 @@ var ( // block hashes to cfilters. cfIndexKeys = [][]byte{ []byte("cf0byhashidx"), // bucket for basic filter indexes - []byte("cf1byhashidx"), // bucket for UtreexoCFilter } // cfHeaderKeys is an array of db bucket names used to house indexes of @@ -48,7 +47,6 @@ var ( // block hashes to cf hashes. cfHashKeys = [][]byte{ []byte("cf0hashbyhashidx"), - []byte("cf1hashbyhashidx"), } maxFilterType = uint8(len(cfHeaderKeys) - 1) diff --git a/blockchain/indexers/flatutreexoproofindex.go b/blockchain/indexers/flatutreexoproofindex.go index 72a77463..c60d6758 100644 --- a/blockchain/indexers/flatutreexoproofindex.go +++ b/blockchain/indexers/flatutreexoproofindex.go @@ -354,7 +354,24 @@ func (idx *FlatUtreexoProofIndex) ConnectBlock(dbTx database.Tx, block *btcutil. } } - return nil + blockHash := block.Hash() + height, err := idx.chain.BlockHeightByHash(blockHash) + if err != nil { + return err + } + + roots, leaves, err := idx.FetchUtreexoState(height) + if err != nil { + return err + } + + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err := blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + return err + } + + return storeUtreexoCFilterHeader(dbTx, block, serializedUtreexo, wire.UtreexoCFilter) } // calcProofOverhead calculates the overhead of the current utreexo accumulator proof diff --git a/blockchain/indexers/utreexoproofindex.go b/blockchain/indexers/utreexoproofindex.go index 67af564c..af333189 100644 --- a/blockchain/indexers/utreexoproofindex.go +++ b/blockchain/indexers/utreexoproofindex.go @@ -238,34 +238,18 @@ func (idx *UtreexoProofIndex) Create(dbTx database.Tx) error { return nil } -// storeFilter stores a given filter, and performs the steps needed to -// generate the filter's header. -func storeUtreexoCFilter(dbTx database.Tx, block *btcutil.Block, filterData []byte, +// storeUtreexoCFilter stores a given utreexocfilter header +func storeUtreexoCFilterHeader(dbTx database.Tx, block *btcutil.Block, filterData []byte, filterType wire.FilterType) error { if uint8(filterType) > maxFilterType { return errors.New("unsupported filter type") } - // Figure out which buckets to use. - fkey := cfIndexKeys[filterType] + // Figure out which header bucket to use. hkey := cfHeaderKeys[filterType] - hashkey := cfHashKeys[filterType] - - // Start by storing the filter. h := block.Hash() - err := dbStoreFilterIdxEntry(dbTx, fkey, h, filterData) - if err != nil { - return err - } - - // Next store the filter hash. - filterHash := chainhash.DoubleHashH(filterData) - err = dbStoreFilterIdxEntry(dbTx, hashkey, h, filterHash[:]) - if err != nil { - return err - } - // Then fetch the previous block's filter header. + // fetch the previous block's filter header. var prevHeader *chainhash.Hash ph := &block.MsgBlock().Header.PrevBlock if ph.IsEqual(&zeroHash) { @@ -381,7 +365,7 @@ func (idx *UtreexoProofIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Bloc return err } - return storeUtreexoCFilter(dbTx, block, serializedUtreexo, wire.UtreexoCFilter) + return storeUtreexoCFilterHeader(dbTx, block, serializedUtreexo, wire.UtreexoCFilter) } // getUndoData returns the data needed for undo. For pruned nodes, we fetch the data from diff --git a/server.go b/server.go index b3ed4793..ff37dddd 100644 --- a/server.go +++ b/server.go @@ -863,172 +863,343 @@ func (sp *serverPeer) OnGetCFilters(_ *peer.Peer, msg *wire.MsgGetCFilters) { return } + var hashes []chainhash.Hash + var hashPtrs []*chainhash.Hash + // if the filter type is supported, we initialize variables to avoid duplicate code + if msg.FilterType == wire.GCSFilterRegular || msg.FilterType == wire.UtreexoCFilter { + var err error + // get the block hashes included in the getcfilters message + hashes, err = sp.server.chain.HeightToHashRange( + int32(msg.StartHeight), &msg.StopHash, wire.MaxGetCFiltersReqRange, + ) + if err != nil { + peerLog.Debugf("Invalid getcfilters request: %v", err) + return + } + + // Create []*chainhash.Hash from []chainhash.Hash to pass to + // FiltersByBlockHashes. + hashPtrs = make([]*chainhash.Hash, len(hashes)) + for i := range hashes { + hashPtrs[i] = &hashes[i] + } + } + // We'll also ensure that the remote party is requesting a set of // filters that we actually currently maintain. switch msg.FilterType { case wire.GCSFilterRegular: + filters, err := sp.server.cfIndex.FiltersByBlockHashes( + hashPtrs, msg.FilterType, + ) + if err != nil { + peerLog.Errorf("Error retrieving cfilters: %v", err) + return + } + + for i, filterBytes := range filters { + if len(filterBytes) == 0 { + peerLog.Warnf("Could not obtain cfilter for %v", + hashes[i]) + return + } + + filterMsg := wire.NewMsgCFilter( + msg.FilterType, &hashes[i], filterBytes, + ) + sp.QueueMessage(filterMsg, nil) + } + case wire.UtreexoCFilter: - break + for i, blockHash := range hashPtrs { + var serializedUtreexo []byte + + leaves, roots, err := sp.getUtreexoRoots(blockHash) + if err != nil { + return + } + + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err = blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + peerLog.Errorf("error serializing utreexoc filter: %v", err) + return + } + + if len(serializedUtreexo) == 0 { + peerLog.Warnf("Could not obtain utreexocfilter for %v", + hashes[i]) + return + } + + filterMsg := wire.NewMsgCFilter( + msg.FilterType, &hashes[i], serializedUtreexo, + ) + sp.QueueMessage(filterMsg, nil) + } default: peerLog.Debug("Filter request for unknown filter: %v", msg.FilterType) return } +} - hashes, err := sp.server.chain.HeightToHashRange( - int32(msg.StartHeight), &msg.StopHash, wire.MaxGetCFiltersReqRange, - ) - if err != nil { - peerLog.Debugf("Invalid getcfilters request: %v", err) +// OnGetCFHeaders is invoked when a peer receives a getcfheader bitcoin message. +func (sp *serverPeer) OnGetCFHeaders(_ *peer.Peer, msg *wire.MsgGetCFHeaders) { + // Ignore getcfilterheader requests if not in sync. + if !sp.server.syncManager.IsCurrent() { return } - // Create []*chainhash.Hash from []chainhash.Hash to pass to - // FiltersByBlockHashes. - hashPtrs := make([]*chainhash.Hash, len(hashes)) - for i := range hashes { - hashPtrs[i] = &hashes[i] - } + var startHeight int32 + var maxResults int + var hashList []chainhash.Hash + var hashPtrs []*chainhash.Hash + // if the filter type is supported, we initialize variables to avoid duplicate code + if msg.FilterType == wire.GCSFilterRegular || msg.FilterType == wire.UtreexoCFilter { - filters, err := sp.server.cfIndex.FiltersByBlockHashes( - hashPtrs, msg.FilterType, - ) - if err != nil { - peerLog.Errorf("Error retrieving cfilters: %v", err) - return - } + startHeight = int32(msg.StartHeight) + maxResults = wire.MaxCFHeadersPerMsg - for i, filterBytes := range filters { - if len(filterBytes) == 0 { - peerLog.Warnf("Could not obtain cfilter for %v", - hashes[i]) - return + // If StartHeight is positive, fetch the predecessor block hash so we + // can populate the PrevFilterHeader field. + if msg.StartHeight > 0 { + startHeight-- + maxResults++ } - filterMsg := wire.NewMsgCFilter( - msg.FilterType, &hashes[i], filterBytes, + // Fetch the hashes from the block index. + var err error + hashList, err = sp.server.chain.HeightToHashRange( + startHeight, &msg.StopHash, maxResults, ) - sp.QueueMessage(filterMsg, nil) - } -} + if err != nil { + peerLog.Debugf("Invalid getcfheaders request: %v", err) + } -// OnGetCFHeaders is invoked when a peer receives a getcfheader bitcoin message. -func (sp *serverPeer) OnGetCFHeaders(_ *peer.Peer, msg *wire.MsgGetCFHeaders) { - // Ignore getcfilterheader requests if not in sync. - if !sp.server.syncManager.IsCurrent() { - return + // This is possible if StartHeight is one greater that the height of + // StopHash, and we pull a valid range of hashes including the previous + // filter header. + if len(hashList) == 0 || (msg.StartHeight > 0 && len(hashList) == 1) { + peerLog.Debug("No results for getcfheaders request") + return + } + + // Create []*chainhash.Hash from []chainhash.Hash to pass to + // FilterHeadersByBlockHashes. + hashPtrs = make([]*chainhash.Hash, len(hashList)) + for i := range hashList { + hashPtrs[i] = &hashList[i] + } } // We'll also ensure that the remote party is requesting a set of // headers for filters that we actually currently maintain. switch msg.FilterType { case wire.GCSFilterRegular: + + // Fetch the raw filter hash bytes from the database for all blocks. + filterHashes, err := sp.server.cfIndex.FilterHashesByBlockHashes( + hashPtrs, msg.FilterType, + ) + if err != nil { + peerLog.Errorf("Error retrieving cfilter hashes: %v", err) + return + } + + // Generate cfheaders message and send it. + headersMsg := wire.NewMsgCFHeaders() + + // Populate the PrevFilterHeader field. + if msg.StartHeight > 0 { + prevBlockHash := &hashList[0] + + // Fetch the raw committed filter header bytes from the + // database. + headerBytes, err := sp.server.cfIndex.FilterHeaderByBlockHash( + prevBlockHash, msg.FilterType) + if err != nil { + peerLog.Errorf("Error retrieving CF header: %v", err) + return + } + if len(headerBytes) == 0 { + peerLog.Warnf("Could not obtain CF header for %v", prevBlockHash) + return + } + + // Deserialize the hash into PrevFilterHeader. + err = headersMsg.PrevFilterHeader.SetBytes(headerBytes) + if err != nil { + peerLog.Warnf("Committed filter header deserialize "+ + "failed: %v", err) + return + } + + hashList = hashList[1:] + filterHashes = filterHashes[1:] + } + + // Populate HeaderHashes. + for i, hashBytes := range filterHashes { + if len(hashBytes) == 0 { + peerLog.Warnf("Could not obtain CF hash for %v", hashList[i]) + return + } + + // Deserialize the hash. + filterHash, err := chainhash.NewHash(hashBytes) + if err != nil { + peerLog.Warnf("Committed filter hash deserialize "+ + "failed: %v", err) + return + } + + headersMsg.AddCFHash(filterHash) + } + + headersMsg.FilterType = msg.FilterType + headersMsg.StopHash = msg.StopHash + + sp.QueueMessage(headersMsg, nil) + + // handle custom utreexocfilter message case wire.UtreexoCFilter: - break - default: - peerLog.Debug("Filter request for unknown headers for "+ - "filter: %v", msg.FilterType) - return - } + // Generate cfheaders message and send it. + headersMsg := wire.NewMsgCFHeaders() - startHeight := int32(msg.StartHeight) - maxResults := wire.MaxCFHeadersPerMsg + // Populate the PrevFilterHeader field. + if msg.StartHeight > 0 { + prevBlockHash := &hashList[0] - // If StartHeight is positive, fetch the predecessor block hash so we - // can populate the PrevFilterHeader field. - if msg.StartHeight > 0 { - startHeight-- - maxResults++ - } + // Fetch the raw committed filter header bytes from the + // database. + headerBytes, err := sp.server.cfIndex.FilterHeaderByBlockHash( + prevBlockHash, msg.FilterType) + if err != nil { + peerLog.Errorf("Error retrieving CF header: %v", err) + return + } + if len(headerBytes) == 0 { + peerLog.Warnf("Could not obtain CF header for %v", prevBlockHash) + return + } - // Fetch the hashes from the block index. - hashList, err := sp.server.chain.HeightToHashRange( - startHeight, &msg.StopHash, maxResults, - ) - if err != nil { - peerLog.Debugf("Invalid getcfheaders request: %v", err) - } + // Deserialize the hash into PrevFilterHeader. + err = headersMsg.PrevFilterHeader.SetBytes(headerBytes) + if err != nil { + peerLog.Warnf("Committed filter header deserialize "+ + "failed: %v", err) + return + } + } - // This is possible if StartHeight is one greater that the height of - // StopHash, and we pull a valid range of hashes including the previous - // filter header. - if len(hashList) == 0 || (msg.StartHeight > 0 && len(hashList) == 1) { - peerLog.Debug("No results for getcfheaders request") - return - } + // fetch filter hashes and add to cf hashes field + for i, blockHash := range hashPtrs { + var serializedUtreexo []byte + // skip the first index as this index was added so as to enable us + // to get the previous filter's header + if i == 0 { + continue + } - // Create []*chainhash.Hash from []chainhash.Hash to pass to - // FilterHeadersByBlockHashes. - hashPtrs := make([]*chainhash.Hash, len(hashList)) - for i := range hashList { - hashPtrs[i] = &hashList[i] - } + leaves, roots, err := sp.getUtreexoRoots(blockHash) + if err != nil { + return + } - // Fetch the raw filter hash bytes from the database for all blocks. - filterHashes, err := sp.server.cfIndex.FilterHashesByBlockHashes( - hashPtrs, msg.FilterType, - ) - if err != nil { - peerLog.Errorf("Error retrieving cfilter hashes: %v", err) + // serialize the hashes of the utreexo roots hash + serializedUtreexo, err = blockchain.SerializeUtreexoRootsHash(leaves, roots) + if err != nil { + peerLog.Errorf("error serializing utreexoc filter: %v", err) + return + } + + if len(serializedUtreexo) == 0 { + peerLog.Warnf("Could not obtain utreexocfilter for %v", + hashList[i]) + return + } + hashBytes := chainhash.DoubleHashB(serializedUtreexo) + + if len(hashBytes) == 0 { + peerLog.Warnf("Could not obtain CF hash for %v", hashList[i]) + return + } + + // Deserialize the hash. + filterHash, err := chainhash.NewHash(hashBytes) + if err != nil { + peerLog.Warnf("Committed filter hash deserialize "+ + "failed: %v", err) + return + } + + headersMsg.AddCFHash(filterHash) + } + headersMsg.FilterType = msg.FilterType + headersMsg.StopHash = msg.StopHash + + sp.QueueMessage(headersMsg, nil) + + default: + peerLog.Debug("Filter request for unknown headers for "+ + "filter: %v", msg.FilterType) return } +} - // Generate cfheaders message and send it. - headersMsg := wire.NewMsgCFHeaders() +func (sp *serverPeer) getUtreexoRoots(blockHash *chainhash.Hash) (uint64, []*chainhash.Hash, error) { - // Populate the PrevFilterHeader field. - if msg.StartHeight > 0 { - prevBlockHash := &hashList[0] + var leaves uint64 + var roots []*chainhash.Hash - // Fetch the raw committed filter header bytes from the - // database. - headerBytes, err := sp.server.cfIndex.FilterHeaderByBlockHash( - prevBlockHash, msg.FilterType) + // For compact state nodes + if !cfg.NoUtreexo { + viewPoint, err := sp.server.chain.FetchUtreexoViewpoint(blockHash) if err != nil { - peerLog.Errorf("Error retrieving CF header: %v", err) - return - } - if len(headerBytes) == 0 { - peerLog.Warnf("Could not obtain CF header for %v", prevBlockHash) - return + peerLog.Errorf("could not obtain utreexo view: %v", err) + return 0, nil, err } + roots = viewPoint.GetRoots() + leaves = viewPoint.NumLeaves() + } + // for bridge nodes + if sp.server.utreexoProofIndex != nil { + var uleaves uint64 + var uroots []*chainhash.Hash + var err error + err = sp.server.db.View(func(dbTx database.Tx) error { + uroots, uleaves, err = sp.server.utreexoProofIndex.FetchUtreexoState(dbTx, blockHash) + if err != nil { + return err + } - // Deserialize the hash into PrevFilterHeader. - err = headersMsg.PrevFilterHeader.SetBytes(headerBytes) + return nil + }) if err != nil { - peerLog.Warnf("Committed filter header deserialize "+ - "failed: %v", err) - return + peerLog.Errorf("error fetching utreexo view for blockhash %s: error: %v", blockHash, err) + return 0, nil, err } - - hashList = hashList[1:] - filterHashes = filterHashes[1:] - } - - // Populate HeaderHashes. - for i, hashBytes := range filterHashes { - if len(hashBytes) == 0 { - peerLog.Warnf("Could not obtain CF hash for %v", hashList[i]) - return + roots = uroots + leaves = uleaves + } else if sp.server.flatUtreexoProofIndex != nil { + height, err := sp.server.chain.BlockHeightByHash(blockHash) + if err != nil { + peerLog.Errorf("couldn't fetch the block height for blockhash %s from "+ + "the blockindex. Error: %v", blockHash, err) + return 0, nil, err } - - // Deserialize the hash. - filterHash, err := chainhash.NewHash(hashBytes) + uroots, uleaves, err := sp.server.flatUtreexoProofIndex.FetchUtreexoState(height) if err != nil { - peerLog.Warnf("Committed filter hash deserialize "+ - "failed: %v", err) - return + peerLog.Errorf("error fetching utreexo view for blockhash: %s: error: %v", err) + return 0, nil, err } - - headersMsg.AddCFHash(filterHash) + roots = uroots + leaves = uleaves } - - headersMsg.FilterType = msg.FilterType - headersMsg.StopHash = msg.StopHash - - sp.QueueMessage(headersMsg, nil) + return leaves, roots, nil } // OnGetCFCheckpt is invoked when a peer receives a getcfcheckpt bitcoin message.