From f6b5b1134c06e88c80a90c20e64452424633d7ce Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 22 Aug 2020 18:12:04 +0200 Subject: [PATCH 1/2] core: do less lookups when writing fast-sync block bodies (#21468) --- core/blockchain.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 8c2f61e4d8..9af77c2192 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1256,6 +1256,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // writeLive writes blockchain and corresponding receipt chain into active store. writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { + skipPresenceCheck := false batch := bc.db.NewBatch() for i, block := range blockChain { // Short circuit insertion if shutting down or processing failed @@ -1266,9 +1267,17 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if !bc.HasHeader(block.Hash(), block.NumberU64()) { return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) } - if bc.HasBlock(block.Hash(), block.NumberU64()) { - stats.ignored++ - continue + if !skipPresenceCheck { + // Ignore if the entire data is already known + if bc.HasBlock(block.Hash(), block.NumberU64()) { + stats.ignored++ + continue + } else { + // If block N is not present, neither are the later blocks. + // This should be true, but if we are mistaken, the shortcut + // here will only cause overwriting of some existing data + skipPresenceCheck = true + } } // Write all the data out into the database rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()) From ef92ef80589a78a60dff346d76a68cacdbb64f7b Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 24 Aug 2020 10:10:18 +0800 Subject: [PATCH 2/2] download for short chain --- ctxc/downloader/downloader.go | 2 ++ go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/CortexFoundation/torrentfs/files.go | 1 + vendor/github.com/CortexFoundation/torrentfs/fs.go | 7 +++++++ vendor/github.com/CortexFoundation/torrentfs/handler.go | 4 ++-- vendor/modules.txt | 2 +- 7 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ctxc/downloader/downloader.go b/ctxc/downloader/downloader.go index 6949ade854..9078d1204d 100644 --- a/ctxc/downloader/downloader.go +++ b/ctxc/downloader/downloader.go @@ -1432,6 +1432,8 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er head := chunk[len(chunk)-1].Number.Uint64() if head-rollback > uint64(fsHeaderSafetyNet) { rollback = head - uint64(fsHeaderSafetyNet) + } else { + rollback = 1 } } // Unless we're doing light chains, schedule the headers for associated content retrieval diff --git a/go.mod b/go.mod index 8d30b1086d..d17b895174 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/Azure/azure-storage-blob-go v0.8.1-0.20191213204130-762620a866ba github.com/CortexFoundation/inference v0.0.0-20200819124612-ecf4b33bd901 - github.com/CortexFoundation/torrentfs v1.0.20-0.20200819053011-10bee23abf91 + github.com/CortexFoundation/torrentfs v1.0.20-0.20200821074238-37144679d7c8 github.com/VictoriaMetrics/fastcache v1.5.8-0.20200305212624-8835719dc76c github.com/aristanetworks/goarista v0.0.0-20200513152637-638451432ae4 github.com/arsham/figurine v1.0.1 diff --git a/go.sum b/go.sum index 00faf5c260..b9e06f0610 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/CortexFoundation/torrentfs v1.0.14-0.20200703071639-3fcabcabf274/go.m github.com/CortexFoundation/torrentfs v1.0.20-0.20200810031954-d36d26f82fcc/go.mod h1:N5BsicP5ynjXIi/Npl/SRzlJ630n1PJV2sRj0Z0t2HA= github.com/CortexFoundation/torrentfs v1.0.20-0.20200814090747-6d2d8ab19a8d h1:V1rOyN8MsrziwAenKE6DYIuskQSPkzV5Pu7nzy/pkEM= github.com/CortexFoundation/torrentfs v1.0.20-0.20200814090747-6d2d8ab19a8d/go.mod h1:N5BsicP5ynjXIi/Npl/SRzlJ630n1PJV2sRj0Z0t2HA= -github.com/CortexFoundation/torrentfs v1.0.20-0.20200819053011-10bee23abf91 h1:qdEKayqBG4ikRrhnshS9kSlqYgdm1RAQwecQ4xLegzo= -github.com/CortexFoundation/torrentfs v1.0.20-0.20200819053011-10bee23abf91/go.mod h1:N5BsicP5ynjXIi/Npl/SRzlJ630n1PJV2sRj0Z0t2HA= +github.com/CortexFoundation/torrentfs v1.0.20-0.20200821074238-37144679d7c8 h1:w+lqlUVjlDMr+RlG5csIe+cEpFGmfXXP905m303xAAY= +github.com/CortexFoundation/torrentfs v1.0.20-0.20200821074238-37144679d7c8/go.mod h1:N5BsicP5ynjXIi/Npl/SRzlJ630n1PJV2sRj0Z0t2HA= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= diff --git a/vendor/github.com/CortexFoundation/torrentfs/files.go b/vendor/github.com/CortexFoundation/torrentfs/files.go index 418dfc2fed..0b05fcd9da 100644 --- a/vendor/github.com/CortexFoundation/torrentfs/files.go +++ b/vendor/github.com/CortexFoundation/torrentfs/files.go @@ -18,6 +18,7 @@ package torrentfs var BadFiles = map[string]bool{ "3edcb8a793887d92db12d53124955681d5c20a43": true, + //"7c6ff2ae3b9dfc654c5dec61cc950b70a16431c4": true, } var GoodFiles = map[string]bool{ diff --git a/vendor/github.com/CortexFoundation/torrentfs/fs.go b/vendor/github.com/CortexFoundation/torrentfs/fs.go index a5d5501ee2..04f66f7a10 100644 --- a/vendor/github.com/CortexFoundation/torrentfs/fs.go +++ b/vendor/github.com/CortexFoundation/torrentfs/fs.go @@ -43,6 +43,7 @@ type TorrentFS struct { nasCache *lru.Cache queryCache *lru.Cache + nasCounter uint64 } func (t *TorrentFS) storage() *TorrentManager { @@ -102,6 +103,7 @@ func New(config *Config, cache, compress, listen bool) (*TorrentFS, error) { "number": monitor.currentNumber, "maxMessageSize": inst.MaxMessageSize(), // "listen": monitor.listen, + "metrics": inst.NasCounter(), }, } }, @@ -191,6 +193,7 @@ func (tfs *TorrentFS) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { return err } } + tfs.nasCounter++ tfs.queryCache.Add(info.Hash, info.Size) } } @@ -330,3 +333,7 @@ func (fs *TorrentFS) Congress() int { func (fs *TorrentFS) Candidate() int { return fs.storage().Candidate() } + +func (fs *TorrentFS) NasCounter() uint64 { + return fs.nasCounter +} diff --git a/vendor/github.com/CortexFoundation/torrentfs/handler.go b/vendor/github.com/CortexFoundation/torrentfs/handler.go index 5b909c5915..1b433ca00f 100644 --- a/vendor/github.com/CortexFoundation/torrentfs/handler.go +++ b/vendor/github.com/CortexFoundation/torrentfs/handler.go @@ -834,10 +834,10 @@ func (tm *TorrentManager) dropSeeding(slot int) error { } if tm.mode == LAZY { - t.currentConns = 0 + t.currentConns = 1 log.Debug("Lazy mode dropped", "ih", ih, "seeding", len(tm.seedingTorrents), "torrents", len(tm.torrents), "max", tm.maxSeedTask, "peers", t.currentConns, "cited", t.cited) } else { - t.currentConns = 1 + t.currentConns = 2 } t.Torrent.SetMaxEstablishedConns(t.currentConns) log.Debug("Drop seeding invoke", "ih", ih, "index", i, "group", s, "slot", slot, "len", len(tm.seedingTorrents), "max", tm.maxSeedTask, "peers", t.currentConns, "cited", t.cited) diff --git a/vendor/modules.txt b/vendor/modules.txt index cc34b500e6..e193f9be2b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -13,7 +13,7 @@ github.com/CortexFoundation/cvm-runtime/kernel ## explicit github.com/CortexFoundation/inference github.com/CortexFoundation/inference/synapse -# github.com/CortexFoundation/torrentfs v1.0.20-0.20200819053011-10bee23abf91 +# github.com/CortexFoundation/torrentfs v1.0.20-0.20200821074238-37144679d7c8 ## explicit github.com/CortexFoundation/torrentfs github.com/CortexFoundation/torrentfs/compress