From b9869663616be4ac56a1222841265dc0b696c253 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Fri, 6 Nov 2015 10:49:46 -0800 Subject: [PATCH 1/2] add WriteToFlag to Tx For in memory workload, it does not make sense to use o_direct to copy the file. Adding a option to clear out o_direct and for other future cases. --- bolt_linux.go | 2 -- bolt_openbsd.go | 2 -- bolt_windows.go | 2 -- tx.go | 19 +++++++++++++------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bolt_linux.go b/bolt_linux.go index e9d1c907..2b676661 100644 --- a/bolt_linux.go +++ b/bolt_linux.go @@ -4,8 +4,6 @@ import ( "syscall" ) -var odirect = syscall.O_DIRECT - // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { return syscall.Fdatasync(int(db.file.Fd())) diff --git a/bolt_openbsd.go b/bolt_openbsd.go index 7c1bef1a..7058c3d7 100644 --- a/bolt_openbsd.go +++ b/bolt_openbsd.go @@ -11,8 +11,6 @@ const ( msInvalidate // invalidate cached data ) -var odirect int - func msync(db *DB) error { _, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(db.data)), uintptr(db.datasz), msInvalidate) if errno != 0 { diff --git a/bolt_windows.go b/bolt_windows.go index f0483655..91c4968f 100644 --- a/bolt_windows.go +++ b/bolt_windows.go @@ -40,8 +40,6 @@ func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *sysc return nil } -var odirect int - // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { return db.file.Sync() diff --git a/tx.go b/tx.go index fe6c287f..e181aab6 100644 --- a/tx.go +++ b/tx.go @@ -29,6 +29,16 @@ type Tx struct { pages map[pgid]*page stats TxStats commitHandlers []func() + + // WriteFlag specifies the flag for write related methods + // like WriteTo. + // Tx opens the database file with the specified + // flag to copy the data. + // + // By default, the flag is set to empty for in-memory workload. + // To avoid cache trashing for large on disk workload, set this + // flag with o_direct. + WriteFlag int } // init initializes the transaction. @@ -272,13 +282,10 @@ func (tx *Tx) Copy(w io.Writer) error { // WriteTo writes the entire database to a writer. // If err == nil then exactly tx.Size() bytes will be written into the writer. func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { - // Attempt to open reader directly. + // Attempt to open reader with WriteFlag var f *os.File - if f, err = os.OpenFile(tx.db.path, os.O_RDONLY|odirect, 0); err != nil { - // Fallback to a regular open if that doesn't work. - if f, err = os.OpenFile(tx.db.path, os.O_RDONLY, 0); err != nil { - return 0, err - } + if f, err = os.OpenFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0); err != nil { + return 0, err } // Copy the meta pages. From 6b1bbf0ab40ce12920ca9ad28466107a18f47036 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 6 Nov 2015 13:18:58 -0700 Subject: [PATCH 2/2] update WriteTo() docs This commit updates and revises some of the documentation around Tx.WriteTo() and how O_DIRECT is no longer the default. --- README.md | 7 +++++-- boltsync_unix.go | 2 -- tx.go | 19 ++++++++----------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 32759ce9..80353ab5 100644 --- a/README.md +++ b/README.md @@ -426,8 +426,11 @@ func (*Bucket) DeleteBucket(key []byte) error Bolt is a single file so it's easy to backup. You can use the `Tx.WriteTo()` function to write a consistent view of the database to a writer. If you call this from a read-only transaction, it will perform a hot backup and not block -your other database reads and writes. It will also use `O_DIRECT` when available -to prevent page cache trashing. +your other database reads and writes. + +By default, it will use a regular file handle which will utilize the operating +system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx) +documentation for information about optimizing for larger-than-RAM datasets. One common use case is to backup over HTTP so you can use tools like `cURL` to do database backups: diff --git a/boltsync_unix.go b/boltsync_unix.go index 8db89776..f5044252 100644 --- a/boltsync_unix.go +++ b/boltsync_unix.go @@ -2,8 +2,6 @@ package bolt -var odirect int - // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { return db.file.Sync() diff --git a/tx.go b/tx.go index e181aab6..3273106e 100644 --- a/tx.go +++ b/tx.go @@ -30,14 +30,12 @@ type Tx struct { stats TxStats commitHandlers []func() - // WriteFlag specifies the flag for write related methods - // like WriteTo. - // Tx opens the database file with the specified - // flag to copy the data. + // WriteFlag specifies the flag for write-related methods like WriteTo(). + // Tx opens the database file with the specified flag to copy the data. // - // By default, the flag is set to empty for in-memory workload. - // To avoid cache trashing for large on disk workload, set this - // flag with o_direct. + // By default, the flag is unset, which works well for mostly in-memory + // workloads. For databases that are much larger than available RAM, + // set the flag to syscall.O_DIRECT to avoid trashing the page cache. WriteFlag int } @@ -283,17 +281,17 @@ func (tx *Tx) Copy(w io.Writer) error { // If err == nil then exactly tx.Size() bytes will be written into the writer. func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { // Attempt to open reader with WriteFlag - var f *os.File - if f, err = os.OpenFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0); err != nil { + f, err := os.OpenFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0) + if err != nil { return 0, err } + defer f.Close() // Copy the meta pages. tx.db.metalock.Lock() n, err = io.CopyN(w, f, int64(tx.db.pageSize*2)) tx.db.metalock.Unlock() if err != nil { - _ = f.Close() return n, fmt.Errorf("meta copy: %s", err) } @@ -301,7 +299,6 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { wn, err := io.CopyN(w, f, tx.Size()-int64(tx.db.pageSize*2)) n += wn if err != nil { - _ = f.Close() return n, err }