Improvement in bbolt/walletdb speed

blockchain/common_test.go

@@ -23,15 +23,12 @@ import (
 	"github.com/pkt-cash/pktd/chaincfg/chainhash"
 	"github.com/pkt-cash/pktd/chaincfg/genesis"
 	"github.com/pkt-cash/pktd/database"
-	_ "github.com/pkt-cash/pktd/database/ffldb"
+	"github.com/pkt-cash/pktd/database/ffldb"
 	"github.com/pkt-cash/pktd/txscript"
 	"github.com/pkt-cash/pktd/wire"
 )
 
 const (
-	// testDbType is the database backend type to use for the tests.
-	testDbType = "ffldb"
-
 	// testDbRoot is the root directory used to create all test databases.
 	testDbRoot = "testdbs"
 
@@ -49,19 +46,6 @@ func fileExists(name string) bool {
 	return true
 }
 
-// isSupportedDbType returns whether or not the passed database type is
-// currently supported.
-func isSupportedDbType(dbType string) bool {
-	supportedDrivers := database.SupportedDrivers()
-	for _, driver := range supportedDrivers {
-		if dbType == driver {
-			return true
-		}
-	}
-
-	return false
-}
-
 // loadBlocks reads files containing bitcoin block data (gzipped but otherwise
 // in the format bitcoind writes) from disk and returns them as an array of
 // btcutil.Block.  This is largely borrowed from the test code in pktdb.
@@ -73,27 +57,10 @@ func loadBlocks(filename string) (blocks []*btcutil.Block, err er.R) {
 // block already inserted.  In addition to the new chain instance, it returns
 // a teardown function the caller should invoke when done testing to clean up.
 func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er.R) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, er.Errorf("unsupported db type %v", testDbType)
-	}
-
 	// Handle memory database specially since it doesn't need the disk
 	// specific handling.
 	var db database.DB
 	var teardown func()
-	if testDbType == "memdb" {
-		ndb, err := database.Create(testDbType)
-		if err != nil {
-			return nil, nil, er.Errorf("error creating db: %v", err)
-		}
-		db = ndb
-
-		// Setup a teardown function for cleaning up.  This function is
-		// returned to the caller to be invoked when it is done testing.
-		teardown = func() {
-			db.Close()
-		}
-	} else {
 		// Create the root directory for test databases.
 		if !fileExists(testDbRoot) {
 			if err := os.MkdirAll(testDbRoot, 0700); err != nil {
@@ -106,7 +73,7 @@ func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er
 		// Create a new database to store the accepted blocks into.
 		dbPath := filepath.Join(testDbRoot, dbName)
 		_ = os.RemoveAll(dbPath)
-		ndb, err := database.Create(testDbType, dbPath, blockDataNet)
+		ndb, err := ffldb.OpenDB(dbPath, blockDataNet, true)
 		if err != nil {
 			return nil, nil, er.Errorf("error creating db: %v", err)
 		}
@@ -119,7 +86,6 @@ func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), er
 			os.RemoveAll(dbPath)
 			os.RemoveAll(testDbRoot)
 		}
-	}
 
 	// Copy the chain params to ensure any modifications the tests do to
 	// the chain parameters do not affect the global instance.

blockchain/example_test.go

@@ -14,8 +14,7 @@ import (
 	"github.com/pkt-cash/pktd/btcutil"
 	"github.com/pkt-cash/pktd/chaincfg"
 	"github.com/pkt-cash/pktd/chaincfg/genesis"
-	"github.com/pkt-cash/pktd/database"
-	_ "github.com/pkt-cash/pktd/database/ffldb"
+	"github.com/pkt-cash/pktd/database/ffldb"
 )
 
 // This example demonstrates how to create a new chain instance and use
@@ -31,7 +30,7 @@ func ExampleBlockChain_ProcessBlock() {
 	// around.
 	dbPath := filepath.Join(os.TempDir(), "exampleprocessblock")
 	_ = os.RemoveAll(dbPath)
-	db, err := database.Create("ffldb", dbPath, chaincfg.MainNetParams.Net)
+	db, err := ffldb.OpenDB(dbPath, chaincfg.MainNetParams.Net, true)
 	if err != nil {
 		fmt.Printf("Failed to create database: %v\n", err)
 		return

blockchain/fullblocks_test.go

@@ -21,15 +21,12 @@ import (
 	"github.com/pkt-cash/pktd/chaincfg"
 	"github.com/pkt-cash/pktd/chaincfg/chainhash"
 	"github.com/pkt-cash/pktd/database"
-	_ "github.com/pkt-cash/pktd/database/ffldb"
+	"github.com/pkt-cash/pktd/database/ffldb"
 	"github.com/pkt-cash/pktd/txscript"
 	"github.com/pkt-cash/pktd/wire"
 )
 
 const (
-	// testDbType is the database backend type to use for the tests.
-	testDbType = "ffldb"
-
 	// testDbRoot is the root directory used to create all test databases.
 	testDbRoot = "testdbs"
 
@@ -47,44 +44,15 @@ func fileExists(name string) bool {
 	return true
 }
 
-// isSupportedDbType returns whether or not the passed database type is
-// currently supported.
-func isSupportedDbType(dbType string) bool {
-	supportedDrivers := database.SupportedDrivers()
-	for _, driver := range supportedDrivers {
-		if dbType == driver {
-			return true
-		}
-	}
-
-	return false
-}
-
 // chainSetup is used to create a new db and chain instance with the genesis
 // block already inserted.  In addition to the new chain instance, it returns
 // a teardown function the caller should invoke when done testing to clean up.
 func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, func(), er.R) {
-	if !isSupportedDbType(testDbType) {
-		return nil, nil, er.Errorf("unsupported db type %v", testDbType)
-	}
 
 	// Handle memory database specially since it doesn't need the disk
 	// specific handling.
 	var db database.DB
 	var teardown func()
-	if testDbType == "memdb" {
-		ndb, err := database.Create(testDbType)
-		if err != nil {
-			return nil, nil, er.Errorf("error creating db: %v", err)
-		}
-		db = ndb
-
-		// Setup a teardown function for cleaning up.  This function is
-		// returned to the caller to be invoked when it is done testing.
-		teardown = func() {
-			db.Close()
-		}
-	} else {
 		// Create the root directory for test databases.
 		if !fileExists(testDbRoot) {
 			if err := os.MkdirAll(testDbRoot, 0700); err != nil {
@@ -97,7 +65,7 @@ func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain,
 		// Create a new database to store the accepted blocks into.
 		dbPath := filepath.Join(testDbRoot, dbName)
 		_ = os.RemoveAll(dbPath)
-		ndb, err := database.Create(testDbType, dbPath, blockDataNet)
+		ndb, err := ffldb.OpenDB(dbPath, blockDataNet, true)
 		if err != nil {
 			return nil, nil, er.Errorf("error creating db: %v", err)
 		}
@@ -110,7 +78,6 @@ func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain,
 			os.RemoveAll(dbPath)
 			os.RemoveAll(testDbRoot)
 		}
-	}
 
 	// Copy the chain params to ensure any modifications the tests do to
 	// the chain parameters do not affect the global instance.

btcutil/certgen_test.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/pkt-cash/pktd/btcutil"
-	//"github.com/davecgh/go-spew/spew"
 )
 
 // TestNewTLSCertPair ensures the NewTLSCertPair function works as expected.

config.go

@@ -26,7 +26,6 @@ import (
 	"github.com/pkt-cash/pktd/chaincfg"
 	"github.com/pkt-cash/pktd/chaincfg/chainhash"
 	"github.com/pkt-cash/pktd/chaincfg/globalcfg"
-	"github.com/pkt-cash/pktd/database"
 	_ "github.com/pkt-cash/pktd/database/ffldb"
 	"github.com/pkt-cash/pktd/mempool"
 	"github.com/pkt-cash/pktd/peer"
@@ -45,7 +44,6 @@ const (
 	defaultMaxRPCClients         = 10
 	defaultMaxRPCWebsockets      = 25
 	defaultMaxRPCConcurrentReqs  = 20
-	defaultDbType                = "ffldb"
 	defaultFreeTxRelayLimit      = 15.0
 	defaultTrickleInterval       = peer.DefaultTrickleInterval
 	defaultBlockMinSize          = 0
@@ -68,7 +66,6 @@ var (
 	defaultHomeDir     = btcutil.AppDataDir("pktd", false)
 	defaultConfigFile  = filepath.Join(defaultHomeDir, defaultConfigFilename)
 	defaultDataDir     = filepath.Join(defaultHomeDir, defaultDataDirname)
-	knownDbTypes       = database.SupportedDrivers()
 	defaultRPCKeyFile  = filepath.Join(defaultHomeDir, "rpc.key")
 	defaultRPCCertFile = filepath.Join(defaultHomeDir, "rpc.cert")
 	defaultLogDir      = filepath.Join(defaultHomeDir, defaultLogDirname)
@@ -131,7 +128,6 @@ type config struct {
 	SimNet               bool          `long:"simnet" description:"Use the simulation test network"`
 	AddCheckpoints       []string      `long:"addcheckpoint" description:"Add a custom checkpoint.  Format: '<height>:<hash>'"`
 	DisableCheckpoints   bool          `long:"nocheckpoints" description:"Disable built-in checkpoints.  Don't do this unless you know what you're doing."`
-	DbType               string        `long:"dbtype" description:"Database backend to use for the Block Chain"`
 	StatsViz			 string		   `long:"statsviz" description:"Enable StatsViz runtime visualization on given port -- NOTE port must be between 1024 and 65535"`
 	Profile              string        `long:"profile" description:"Enable HTTP profiling on given port -- NOTE port must be between 1024 and 65535"`
 	CPUProfile           string        `long:"cpuprofile" description:"Write CPU profile to the specified file"`
@@ -275,17 +271,6 @@ func parseAndSetDebugLevels(debugLevel string) er.R {
 	return nil
 }
 
-// validDbType returns whether or not dbType is a supported database type.
-func validDbType(dbType string) bool {
-	for _, knownType := range knownDbTypes {
-		if dbType == knownType {
-			return true
-		}
-	}
-
-	return false
-}
-
 // removeDuplicateAddresses returns a new slice with all duplicate entries in
 // addrs removed.
 func removeDuplicateAddresses(addrs []string) []string {
@@ -413,7 +398,6 @@ func loadConfig() (*config, []string, er.R) {
 		HomeDir:              defaultHomeDir,
 		DataDir:              defaultDataDir,
 		LogDir:               defaultLogDir,
-		DbType:               defaultDbType,
 		RPCKey:               defaultRPCKeyFile,
 		RPCCert:              defaultRPCCertFile,
 		MinRelayTxFee:        -1, // this gets configured later
@@ -614,16 +598,6 @@ func loadConfig() (*config, []string, er.R) {
 		return nil, nil, err
 	}
 
-	// Validate database type.
-	if !validDbType(cfg.DbType) {
-		str := "%s: The specified database type [%v] is invalid -- " +
-			"supported types %v"
-		err := er.Errorf(str, funcName, cfg.DbType, knownDbTypes)
-		fmt.Fprintln(os.Stderr, err)
-		fmt.Fprintln(os.Stderr, usageMessage)
-		return nil, nil, err
-	}
-
 	// Validate profile port number
 	if cfg.Profile != "" {
 		profilePort, err := strconv.Atoi(cfg.Profile)

connmgr/connmanager.go

@@ -214,14 +214,14 @@ func (cm *ConnManager) handleFailedConn(c *ConnReq) {
 			log.Debugf("Max failed connection attempts reached: [%d] "+
 				"-- retrying connection in: %v", maxFailedAttempts,
 				cm.cfg.RetryDuration)
-			theId := c.id
+			theID := c.id
 			time.AfterFunc(cm.cfg.RetryDuration, func() {
-				cm.Remove(theId)
+				cm.Remove(theID)
 				cm.NewConnReq()
 			})
 		} else {
-			go func(theId uint64) {
-				cm.Remove(theId)
+			go func(theID uint64) {
+				cm.Remove(theID)
 				cm.NewConnReq()
 			}(c.id)
 		}
@@ -428,8 +428,8 @@ func (cm *ConnManager) Connect(c *ConnReq) {
 	// this connection was already canceled
 	if c.State() == ConnCanceled {
 		log.Infof("Ignoring canceled connreq=%v, attempting new connection.", c)
-		theId := c.id
-		cm.Remove(theId)
+		theID := c.id
+		cm.Remove(theID)
 		cm.NewConnReq()
 		return
 	}

database/README.md

@@ -14,12 +14,9 @@ When a client wants programmatic access to the data provided by pktd, they'll
 likely want to use the [rpcclient](https://github.com/pkt-cash/pktd/tree/master/rpcclient)
 package which makes use of the [JSON-RPC API](https://github.com/pkt-cash/pktd/tree/master/docs/json_rpc_api.md).
 
-However, this package could be extremely useful for any applications requiring
-Bitcoin block storage capabilities.
-
-The default backend, ffldb, has a strong focus on speed, efficiency, and
-robustness.  It makes use of GoLevelDB for the metadata, flat files for
-block storage, and checksums in key areas to ensure data integrity.
+The database backend, ffldb, has a strong focus on speed, efficiency, and
+robustness. It makes use of GoLevelDB for storing metadata, and flat files
+for block storage, with checksums in key areas, to ensure data integrity.
 
 ## Feature Overview
 

database/doc.go

@@ -7,15 +7,6 @@ Package database provides a block and metadata storage database.
 
 Overview
 
-As of Feb 2016, there are over 400,000 blocks in the Bitcoin block chain and
-and over 112 million transactions (which turns out to be over 60GB of data).
-This package provides a database layer to store and retrieve this data in a
-simple and efficient manner.
-
-The default backend, ffldb, has a strong focus on speed, efficiency, and
-robustness.  It makes use leveldb for the metadata, flat files for block
-storage, and strict checksums in key areas to ensure data integrity.
-
 A quick overview of the features database provides are as follows:
 
  - Key/value metadata store