Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

build: update to walletdb/bdb version w/ mmap pre-sizing #4243

Closed
wants to merge 4 commits into from
Closed

build: update to walletdb/bdb version w/ mmap pre-sizing #4243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7a4613a
build: update to walletdb/bdb version w/ mmap pre-sizing
Roasbeef May 4, 2020
bb94b76
multi: update to use new kvdb API
Roasbeef May 5, 2020
6c76351
channeldb/migration13: slight memory optimizations
Roasbeef May 5, 2020
d29f1a8
channeldb/migration13: remove append usage for attemptInfo
Roasbeef May 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion chainntnfs/interface_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
"github.com/btcsuite/btcwallet/chain"
_ "github.com/btcsuite/btcwallet/walletdb/bdb" // Required to auto-register the boltdb walletdb implementation.
"github.com/lightninglabs/neutrino"
"github.com/lightningnetwork/lnd/chainntnfs"
"github.com/lightningnetwork/lnd/chainntnfs/bitcoindnotify"
Expand Down
7 changes: 6 additions & 1 deletion chainntnfs/test_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/btcsuite/btcwallet/chain"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/lightninglabs/neutrino"
"go.etcd.io/bbolt"
)

var (
Expand Down Expand Up @@ -268,7 +269,11 @@ func NewNeutrinoBackend(t *testing.T, minerAddr string) (*neutrino.ChainService,
}

dbName := filepath.Join(spvDir, "neutrino.db")
spvDatabase, err := walletdb.Create("bdb", dbName, true)
spvDatabase, err := walletdb.Create(
"bdb", dbName, &bbolt.Options{
NoFreelistSync: true,
},
)
if err != nil {
os.RemoveAll(spvDir)
t.Fatalf("unable to create walletdb: %v", err)
Expand Down
7 changes: 6 additions & 1 deletion chainregistry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
"github.com/lightningnetwork/lnd/lnwire"
"github.com/lightningnetwork/lnd/routing/chainview"
"go.etcd.io/bbolt"
)

const (
Expand Down Expand Up @@ -733,7 +734,11 @@ func initNeutrinoBackend(chainDir string) (*neutrino.ChainService, func(), error
}

dbName := filepath.Join(dbPath, "neutrino.db")
db, err := walletdb.Create("bdb", dbName, !cfg.SyncFreelist)
db, err := walletdb.Create(
"bdb", dbName, &bbolt.Options{
NoFreelistSync: !cfg.SyncFreelist,
},
)
if err != nil {
return nil, nil, fmt.Errorf("unable to create neutrino "+
"database: %v", err)
Expand Down
11 changes: 9 additions & 2 deletions channeldb/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/lnwire"
"go.etcd.io/bbolt"
)

const (
Expand Down Expand Up @@ -165,7 +166,10 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {

// Specify bbolt freelist options to reduce heap pressure in case the
// freelist grows to be very large.
bdb, err := kvdb.Open(kvdb.BoltBackendName, path, opts.NoFreelistSync)
bboltOpts := &bbolt.Options{
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Q: In the etcd PR NoFreeListSync is an lncfg option. I wonder if we ever intend to actually set this to false? And if no, then perhaps we could default it somewhere (walletdb?) such that it's less hurdle to later on remove them from the code.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we want this to be user-configurable as some users have reported slow startup times on weak machines (mobile, rpi) and opt to not set NoFreeListSync.

NoFreelistSync: opts.NoFreelistSync,
}
bdb, err := kvdb.Open(kvdb.BoltBackendName, path, bboltOpts)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -251,7 +255,10 @@ func createChannelDB(dbPath string) error {
}

path := filepath.Join(dbPath, dbName)
bdb, err := kvdb.Create(kvdb.BoltBackendName, path, true)
bboltOpts := &bbolt.Options{
NoFreelistSync: true,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Though this is just for creating the db, should this be user-specified as well? Setting it one way or another shouldn't give any performance benefit as:

  • if we sync the freelist (this takes time), the next startup upon Open will be shorter
  • if we don't sync the freelist (quick Create time), the next startup upon Open will be longer as it needs to construct the freelist

So both scenarios are quicker in one area and slower in another. Only reason I think would be for consistency.

}
bdb, err := kvdb.Create(kvdb.BoltBackendName, path, bboltOpts)
if err != nil {
return err
}
Expand Down
6 changes: 5 additions & 1 deletion channeldb/forwarding_package_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/lnwire"
"go.etcd.io/bbolt"
)

// TestPkgFilterBruteForce tests the behavior of a pkg filter up to size 1000,
Expand Down Expand Up @@ -806,7 +807,10 @@ func makeFwdPkgDB(t *testing.T, path string) kvdb.Backend { // nolint:unparam
path = filepath.Join(path, "fwdpkg.db")
}

bdb, err := kvdb.Create(kvdb.BoltBackendName, path, true)
bboltOpts := &bbolt.Options{
NoFreelistSync: true,
}
bdb, err := kvdb.Create(kvdb.BoltBackendName, path, bboltOpts)
if err != nil {
t.Fatalf("unable to open boltdb: %v", err)
}
Expand Down
52 changes: 27 additions & 25 deletions channeldb/migration13/migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,22 @@ var (
htlcFailInfoKey = []byte("htlc-fail-info")

byteOrder = binary.BigEndian

// failInfo is an empty fail info value for the legacy pre MPP
// payments.
failInfo = []byte{
// Fail time unknown.
0, 0, 0, 0, 0, 0, 0, 0,

// Zero length wire message.
0,

// Failure reason unknown.
0,

// Failure source index zero.
0, 0, 0, 0,
}
)

// MigrateMPP migrates the payments to a new structure that accommodates for mpp
Expand All @@ -69,6 +85,10 @@ func MigrateMPP(tx kvdb.RwTx) error {
}

// With all keys retrieved, start the migration.
var (
zero [8]byte
attemptID [8]byte
)
for _, k := range paymentKeys {
bucket := paymentsBucket.NestedReadWriteBucket(k)

Expand Down Expand Up @@ -114,15 +134,12 @@ func MigrateMPP(tx kvdb.RwTx) error {
}

// Save attempt id for later use.
attemptID := attemptInfo[:8]

// Discard attempt id. It will become a bucket key in the new
// structure.
attemptInfo = attemptInfo[8:]
copy(attemptID[:], attemptInfo[:8])

// Append unknown (zero) attempt time.
var zero [8]byte
attemptInfo = append(attemptInfo, zero[:]...)
// Copy over the attempt info into a new slice, the last 8
// bytes will be zero to mark a zero attempt time.
newAttemptInfo := make([]byte, len(attemptInfo))
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can use a buffer pool both here and above, assuming the Put method copies the bytes over.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Put method doesn't copy the value bytes over, it does copy the keys over: https://github.com/etcd-io/bbolt/blob/master/node.go#L139

copy(newAttemptInfo, attemptInfo[8:])

// Create bucket that contains all htlcs.
htlcsBucket, err := bucket.CreateBucket(paymentHtlcsBucket)
Expand All @@ -131,13 +148,13 @@ func MigrateMPP(tx kvdb.RwTx) error {
}

// Create an htlc for this attempt.
htlcBucket, err := htlcsBucket.CreateBucket(attemptID)
htlcBucket, err := htlcsBucket.CreateBucket(attemptID[:])
if err != nil {
return err
}

// Save migrated attempt info.
err = htlcBucket.Put(htlcAttemptInfoKey, attemptInfo)
err = htlcBucket.Put(htlcAttemptInfoKey, newAttemptInfo)
if err != nil {
return err
}
Expand Down Expand Up @@ -175,21 +192,6 @@ func MigrateMPP(tx kvdb.RwTx) error {

// The htlc failed. Add htlc fail info with reason unknown. We
// don't have access to the original failure reason anymore.
failInfo := []byte{
// Fail time unknown.
0, 0, 0, 0, 0, 0, 0, 0,

// Zero length wire message.
0,

// Failure reason unknown.
0,

// Failure source index zero.
0, 0, 0, 0,
}

// Save fail info.
err = htlcBucket.Put(htlcFailInfoKey, failInfo)
if err != nil {
return err
Expand Down
11 changes: 9 additions & 2 deletions channeldb/migration_01_to_11/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

"github.com/lightningnetwork/lnd/channeldb/kvdb"
"go.etcd.io/bbolt"
)

const (
Expand Down Expand Up @@ -55,7 +56,10 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {

// Specify bbolt freelist options to reduce heap pressure in case the
// freelist grows to be very large.
bdb, err := kvdb.Open(kvdb.BoltBackendName, path, opts.NoFreelistSync)
bboltOpts := &bbolt.Options{
NoFreelistSync: opts.NoFreelistSync,
}
bdb, err := kvdb.Open(kvdb.BoltBackendName, path, bboltOpts)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -84,7 +88,10 @@ func createChannelDB(dbPath string) error {
}

path := filepath.Join(dbPath, dbName)
bdb, err := kvdb.Create(kvdb.BoltBackendName, path, false)
bboltOpts := &bbolt.Options{
NoFreelistSync: false,
}
bdb, err := kvdb.Create(kvdb.BoltBackendName, path, bboltOpts)
if err != nil {
return err
}
Expand Down
6 changes: 5 additions & 1 deletion channeldb/migtest/migtest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"testing"

"github.com/lightningnetwork/lnd/channeldb/kvdb"
"go.etcd.io/bbolt"
)

// MakeDB creates a new instance of the ChannelDB for testing purposes. A
Expand All @@ -20,7 +21,10 @@ func MakeDB() (kvdb.Backend, func(), error) {
}

dbPath := file.Name()
db, err := kvdb.Open(kvdb.BoltBackendName, dbPath, true)
bboltOpts := &bbolt.Options{
NoFreelistSync: true,
}
db, err := kvdb.Open(kvdb.BoltBackendName, dbPath, bboltOpts)
if err != nil {
return nil, nil, err
}
Expand Down
8 changes: 7 additions & 1 deletion contractcourt/briefcase_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lnwallet"
"go.etcd.io/bbolt"
)

var (
Expand Down Expand Up @@ -112,7 +113,12 @@ func makeTestDB() (kvdb.Backend, func(), error) {
return nil, nil, err
}

db, err := kvdb.Create(kvdb.BoltBackendName, tempDirName+"/test.db", true)
bboltOpts := &bbolt.Options{
NoFreelistSync: true,
}
db, err := kvdb.Create(
kvdb.BoltBackendName, tempDirName+"/test.db", bboltOpts,
)
if err != nil {
return nil, nil, err
}
Expand Down
6 changes: 5 additions & 1 deletion contractcourt/channel_arbitrator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
"go.etcd.io/bbolt"
)

const (
Expand Down Expand Up @@ -394,7 +395,10 @@ func createTestChannelArbitrator(t *testing.T, log ArbitratorLog,
return nil, err
}
dbPath := filepath.Join(dbDir, "testdb")
db, err := kvdb.Create(kvdb.BoltBackendName, dbPath, true)
bboltOpts := &bbolt.Options{
NoFreelistSync: true,
}
db, err := kvdb.Create(kvdb.BoltBackendName, dbPath, bboltOpts)
if err != nil {
return nil, err
}
Expand Down
5 changes: 5 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ require (
github.com/rogpeppe/fastuuid v1.2.0 // indirect
github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02
github.com/urfave/cli v1.18.0
go.etcd.io/bbolt v1.3.3
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d
golang.org/x/net v0.0.0-20190206173232-65e2d4e15006
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect
Expand All @@ -73,3 +74,7 @@ replace git.schwanenlied.me/yawning/bsaes.git => github.com/Yawning/bsaes v0.0.0
replace golang.org/x/crypto => golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67

go 1.12

replace github.com/btcsuite/btcwallet/walletdb v1.3.1 => github.com/roasbeef/btcwallet/walletdb v1.1.1-0.20200504232237-d0cfd3720fbc

replace github.com/btcsuite/btcwallet v0.11.1-0.20200403222202-ada7ca077ebb => github.com/roasbeef/btcwallet v0.11.1-0.20200504232237-d0cfd3720fbc
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2ut
github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
github.com/btcsuite/btcutil/psbt v1.0.2 h1:gCVY3KxdoEVU7Q6TjusPO+GANIwVgr9yTLqM+a6CZr8=
github.com/btcsuite/btcutil/psbt v1.0.2/go.mod h1:LVveMu4VaNSkIRTZu2+ut0HDBRuYjqGocxDMNS1KuGQ=
github.com/btcsuite/btcwallet v0.11.1-0.20200403222202-ada7ca077ebb h1:kkq2SSCy+OrC7GVZLIqutoHVR2yW4SJQdX70jtmuLDI=
github.com/btcsuite/btcwallet v0.11.1-0.20200403222202-ada7ca077ebb/go.mod h1:9fJNm1aXi4q9P5Nk23mmqppCy1Le3f2/JMWj9UXKkCc=
github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 h1:KGHMW5sd7yDdDMkCZ/JpP0KltolFsQcB973brBnfj4c=
github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU=
github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 h1:2VsfS0sBedcM5KmDzRMT3+b6xobqWveZGvjb+jFez5w=
Expand All @@ -43,8 +41,6 @@ github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0/go.mod h1:pauEU8UuMFiThe5PB3
github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
github.com/btcsuite/btcwallet/walletdb v1.2.0 h1:E0+M4jHOToAvGWZ27ew5AaDAHDi6fUiXkjUJUnoEOD0=
github.com/btcsuite/btcwallet/walletdb v1.2.0/go.mod h1:9cwc1Yyg4uvd4ZdfdoMnALji+V9gfWSMfxEdLdR5Vwc=
github.com/btcsuite/btcwallet/walletdb v1.3.1 h1:lW1Ac3F1jJY4K11P+YQtRNcP5jFk27ASfrV7C6mvRU0=
github.com/btcsuite/btcwallet/walletdb v1.3.1/go.mod h1:9cwc1Yyg4uvd4ZdfdoMnALji+V9gfWSMfxEdLdR5Vwc=
github.com/btcsuite/btcwallet/wtxmgr v1.0.0 h1:aIHgViEmZmZfe0tQQqF1xyd2qBqFWxX5vZXkkbjtbeA=
github.com/btcsuite/btcwallet/wtxmgr v1.0.0/go.mod h1:vc4gBprll6BP0UJ+AIGDaySoc7MdAmZf8kelfNb8CFY=
github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 h1:kij1x2aL7VE6gtx8KMIt8PGPgI5GV9LgtHFG5KaEMPY=
Expand Down Expand Up @@ -191,6 +187,10 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/roasbeef/btcwallet v0.11.1-0.20200504232237-d0cfd3720fbc h1:W3U+BYYfg4/xGpsFOYXtuA2G6lKOoeCdYzdbUM9a41M=
github.com/roasbeef/btcwallet v0.11.1-0.20200504232237-d0cfd3720fbc/go.mod h1:cPwCPN+sOeaFebJTeVLQ4Bfwh8dyYX3wqyWdYtyL6sM=
github.com/roasbeef/btcwallet/walletdb v1.1.1-0.20200504232237-d0cfd3720fbc h1:3miUtIw38iJ1kh/BV210RnSVpezbbmKnzHTF3lj7beM=
github.com/roasbeef/btcwallet/walletdb v1.1.1-0.20200504232237-d0cfd3720fbc/go.mod h1:HVMjSxQitIbiaKk9A3eExYS7VQggeS/74J6c9K9Cj/Q=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
Expand Down
6 changes: 5 additions & 1 deletion htlcswitch/decayedlog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
sphinx "github.com/lightningnetwork/lightning-onion"
"github.com/lightningnetwork/lnd/chainntnfs"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"go.etcd.io/bbolt"
)

const (
Expand Down Expand Up @@ -89,8 +90,11 @@ func (d *DecayedLog) Start() error {

// Open the boltdb for use.
var err error
bboltOpts := &bbolt.Options{
NoFreelistSync: true,
}
d.db, err = kvdb.Create(
kvdb.BoltBackendName, d.dbPath, true,
kvdb.BoltBackendName, d.dbPath, bboltOpts,
)
if err != nil {
return fmt.Errorf("could not open boltdb: %v", err)
Expand Down
3 changes: 2 additions & 1 deletion keychain/interface_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/btcsuite/btcwallet/wallet"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/davecgh/go-spew/spew"
"github.com/lightningnetwork/lnd/channeldb/kvdb"

_ "github.com/btcsuite/btcwallet/walletdb/bdb" // Required in order to create the default database.
)
Expand Down Expand Up @@ -86,7 +87,7 @@ func createTestBtcWallet(coinType uint32) (func(), *wallet.Wallet, error) {
// the keys required for signing various contracts.
_, err = baseWallet.Manager.FetchScopedKeyManager(chainKeyScope)
if err != nil {
err := walletdb.Update(baseWallet.Database(), func(tx walletdb.ReadWriteTx) error {
err := kvdb.Update(baseWallet.Database(), func(tx walletdb.ReadWriteTx) error {
addrmgrNs := tx.ReadWriteBucket(waddrmgrNamespaceKey)

_, err := baseWallet.Manager.NewScopedKeyManager(
Expand Down
Loading