Postgres implementation.

.github/workflows/go.yml

@@ -7,6 +7,21 @@ jobs:
     strategy:
       matrix:
         go: [1.14, 1.15]
+
+    services:
+      postgres:
+        image: postgres:13.0
+        env:
+          POSTGRES_USER: dcrpooluser
+          POSTGRES_PASSWORD: 12345
+          POSTGRES_DB: dcrpooltestdb
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
       - name: Set up Go
         uses: actions/setup-go@v2

README.md

@@ -41,17 +41,13 @@ of the address mining rewards are paid to and its name, formatted as:
 the address provided in the username to create an account, all other connected 
 miners with the same address set will contribute work to that account.  
 
-As a contingency, the pool maintains a backup of the database (`backup.kv`), 
-created on shutdown in the same directory as the database itself.
-
 The user interface of the pool provides public access to statistics and pool 
 account data. Users of the pool can access all payments, mined blocks by the 
 account and also work contributed by clients of the account via the interface. 
 The interface is only accessible via HTTPS and by default uses a self-signed 
 certificate, served on port `:8080`. In production, particularly for pool 
 mining, a certificate from an authority (`CA`) like 
-[letsencrypt](https://letsencrypt.org/) is recommended. The user interface also 
-provides pool administrators database backup functionality when needed.
+[letsencrypt](https://letsencrypt.org/) is recommended.
 
 ## Installing and Updating
 
@@ -75,6 +71,17 @@ run `go install . ./cmd/...` in the root directory.  Some notes:
 - The `dcrpool` executable will be installed to `$GOPATH/bin`.  `GOPATH`
   defaults to `$HOME/go` (or `%USERPROFILE%\go` on Windows) if unset.
 
+## Database
+
+dcrpool can run with either a [Bolt database](https://github.com/etcd-io/bbolt)
+or a [Postgres database](https://www.postgresql.org/). Bolt is used by default.
+[postgres.md](./docs/postgres.md) has more details about running with Postgres.
+
+When running in Bolt mode, the pool maintains a backup of the database
+(`backup.kv`), created on shutdown in the same directory as the database itself.
+The user interface also provides functionality for pool administrators to backup
+Bolt database when necessary.
+
 ### Example of obtaining and building from source on Ubuntu
 
 ```sh

config.go

@@ -58,6 +58,12 @@ const (
 	defaultMaxConnectionsPerHost = 100 // 100 connected clients per host
 	defaultWalletAccount         = 0
 	defaultCoinbaseConfTimeout   = time.Minute * 5 // one block time
+	defaultUsePostgres           = false
+	defaultPGHost                = "127.0.0.1"
+	defaultPGPort                = 5432
+	defaultPGUser                = "dcrpooluser"
+	defaultPGPass                = "12345"
+	defaultPGDBName              = "dcrpooldb"
 )
 
 var (
@@ -128,6 +134,12 @@ type config struct {
 	DCR1Port              uint32        `long:"dcr1port" ini-name:"dcr1port" description:"Obelisk DCR1 connection port."`
 	CoinbaseConfTimeout   time.Duration `long:"conftimeout" ini-name:"conftimeout" description:"The duration to wait for coinbase confirmations."`
 	GenCertsOnly          bool          `long:"gencertsonly" ini-name:"gencertsonly" description:"Only generate needed TLS key pairs and terminate."`
+	UsePostgres           bool          `long:"postgres" ini-name:"postgres" description:"Use postgres database instead of bolt."`
+	PGHost                string        `long:"postgreshost" ini-name:"postgreshost" description:"Host to establish a postgres connection."`
+	PGPort                uint32        `long:"postgresport" ini-name:"postgresport" description:"Port to establish a postgres connection."`
+	PGUser                string        `long:"postgresuser" ini-name:"postgresuser" description:"Username for postgres authentication."`
+	PGPass                string        `long:"postgrespass" ini-name:"postgrespass" description:"Password for postgres authentication."`
+	PGDBName              string        `long:"postgresdbname" ini-name:"postgresdbname" description:"Postgres database name."`
 	poolFeeAddrs          []dcrutil.Address
 	dcrdRPCCerts          []byte
 	net                   *params
@@ -362,6 +374,12 @@ func loadConfig() (*config, []string, error) {
 		DCR1Port:              defaultDCR1Port,
 		WalletAccount:         defaultWalletAccount,
 		CoinbaseConfTimeout:   defaultCoinbaseConfTimeout,
+		UsePostgres:           defaultUsePostgres,
+		PGHost:                defaultPGHost,
+		PGPort:                defaultPGPort,
+		PGUser:                defaultPGUser,
+		PGPass:                defaultPGPass,
+		PGDBName:              defaultPGDBName,
 	}
 
 	// Service options which are only added on Windows.

dcrpool.go

@@ -212,13 +212,17 @@ func newPool(db pool.Database, cfg *config) (*miningPool, error) {
 		FetchLastPaymentInfo:  p.hub.FetchLastPaymentInfo,
 		FetchMinedWork:        p.hub.FetchMinedWork,
 		FetchWorkQuotas:       p.hub.FetchWorkQuotas,
-		HTTPBackupDB:          p.hub.HTTPBackupDB,
 		FetchClients:          p.hub.FetchClients,
 		AccountExists:         p.hub.AccountExists,
 		FetchArchivedPayments: p.hub.FetchArchivedPayments,
 		FetchPendingPayments:  p.hub.FetchPendingPayments,
 		FetchCacheChannel:     p.hub.FetchCacheChannel,
 	}
+
+	if !cfg.UsePostgres {
+		gcfg.HTTPBackupDB = p.hub.HTTPBackupDB
+	}
+
 	p.gui, err = gui.NewGUI(gcfg)
 	if err != nil {
 		p.hub.CloseListeners()
@@ -245,7 +249,14 @@ func main() {
 		}
 	}()
 
-	db, err := pool.InitBoltDB(cfg.DBFile)
+	var db pool.Database
+	if cfg.UsePostgres {
+		db, err = pool.InitPostgresDB(cfg.PGHost, cfg.PGPort, cfg.PGUser,
+			cfg.PGPass, cfg.PGDBName)
+	} else {
+		db, err = pool.InitBoltDB(cfg.DBFile)
+	}
+
 	if err != nil {
 		mpLog.Errorf("failed to initialize database: %v", err)
 		os.Exit(1)
@@ -292,11 +303,13 @@ func main() {
 	p.hub.Run(p.ctx)
 
 	// hub.Run() blocks until the pool is fully shut down. When it returns,
-	// write a backup of the DB, and then close the DB.
-	mpLog.Tracef("Backing up database.")
-	err = db.Backup(pool.BoltBackupFile)
-	if err != nil {
-		mpLog.Errorf("failed to write database backup file: %v", err)
+	// write a backup of the DB (if not using postgres), and then close the DB.
+	if !cfg.UsePostgres {
+		mpLog.Tracef("Backing up database.")
+		err = db.Backup(pool.BoltBackupFile)
+		if err != nil {
+			mpLog.Errorf("failed to write database backup file: %v", err)
+		}
 	}
 
 	db.Close()

docs/postgres.md

@@ -0,0 +1,49 @@
+# Running dcrpool with PostgreSQL
+
+Tested with PostgreSQL 13.0.
+
+**Note:** When running in Postgres mode, backups will not be created
+automatically by dcrpool.
+
+## Setup
+
+1. Connect to your instance of PostgreSQL using `psql` to create a new database
+    and a new user for dcrpool.
+    Be sure to substitute the example password `12345` with something more secure.
+
+    ```no-highlight
+    postgres=# CREATE DATABASE dcrpooldb;
+    CREATE DATABASE
+    postgres=# CREATE USER dcrpooluser WITH ENCRYPTED PASSWORD '12345';
+    CREATE ROLE
+    postgres=# GRANT ALL PRIVILEGES ON DATABASE dcrpooldb to dcrpooluser;
+    GRANT
+    ```
+
+1. **Developers only** - if you are modifying code and wish to run the dcrpool
+   test suite, you will need to create an additional database.
+
+    ```no-highlight
+    postgres=# CREATE DATABASE dcrpooltestdb;
+    CREATE DATABASE
+    postgres=# GRANT ALL PRIVILEGES ON DATABASE dcrpooltestdb to dcrpooluser;
+    GRANT
+    ```
+
+1. Add the database connection details to the dcrpool config file.
+
+    ```no-highlight
+    postgres=true
+    postgreshost=127.0.0.1
+    postgresport=5432
+    postgresuser=dcrpooluser
+    postgrespass=12345
+    postgresdbname=dcrpooldb
+    ```
+
+## Tuning
+
+A helpful online tool to determine good settings for your system is called
+[PGTune](https://pgtune.leopard.in.ua/#/). After providing basic information
+about your hardware, PGTune will output a snippet of optimization settings to
+add to your PostgreSQL config.

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7
 	github.com/jrick/logrotate v1.0.0
 	github.com/kr/pretty v0.1.0 // indirect
+	github.com/lib/pq v1.8.0
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1

go.sum

@@ -115,6 +115,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=

gui/admin.go

@@ -22,6 +22,7 @@ type adminPageData struct {
 	ArchivedPayments      []*archivedPayment
 	PendingPaymentsTotal  string
 	PendingPayments       []*pendingPayment
+	BackupAvailable       bool
 }
 
 // adminPage is the handler for "GET /admin". If the current session is
@@ -73,6 +74,7 @@ func (ui *GUI) adminPage(w http.ResponseWriter, r *http.Request) {
 		PendingPayments:       pendingPmts,
 		ArchivedPaymentsTotal: totalArchived,
 		ArchivedPayments:      archivedPmts,
+		BackupAvailable:       ui.cfg.HTTPBackupDB != nil,
 	}
 
 	ui.renderTemplate(w, "admin", pageData)

gui/assets/templates/admin.html

@@ -9,11 +9,12 @@
             <h1 class="mr-auto text-nowrap">Admin Panel</h1>
 
             <div class="row mr-1">
+                {{ if .BackupAvailable }}
                 <form class="p-2" action="/backup" method="post">
                     {{.HeaderData.CSRF}}
                     <button type="submit" class="btn btn-primary btn-small">Backup</button>
                 </form>
-
+                {{ end }}
                 <form class="p-2" action="/logout" method="post">
                     {{.HeaderData.CSRF}}
                     <button type="submit" class="btn btn-primary btn-small">Logout</button>

harness.sh

@@ -29,6 +29,15 @@ MINER_MAX_PROCS=1
 PAYMENT_METHOD="pplns"
 LAST_N_PERIOD=5m
 GUI_DIR="${HARNESS_ROOT}/gui"
+
+# Using postgres requires the DB specified below to exist and contain no data.
+USE_POSTGRES=true
+POSTGRES_HOST=127.0.0.1
+POSTGRES_PORT=5432
+POSTGRES_USER=dcrpooluser
+POSTGRES_PASS=12345
+POSTGRES_DBNAME=dcrpooldb
+
 # CPU_MINING_ADDR is the mining address printed during creation of vwallet.
 # Initial block rewards from `generate` are sent here so vwallet can buy tickets.
 CPU_MINING_ADDR="SsaJxXSymEGroxAiUY9u1mRq1DDWLxn5WhB"
@@ -128,6 +137,12 @@ adminpass=${ADMIN_PASS}
 guidir=${GUI_DIR}
 designation=${TMUX_SESSION}
 profile=6060
+postgres=${USE_POSTGRES}
+postgreshost=${POSTGRES_HOST}
+postgresport=${POSTGRES_PORT}
+postgresuser=${POSTGRES_USER}
+postgrespass=${POSTGRES_PASS}
+postgresdbname=${POSTGRES_DBNAME}
 EOF
 
 cat > "${HARNESS_ROOT}/mwallet/dcrmwctl.conf" <<EOF

pool/acceptedwork.go

@@ -61,6 +61,7 @@ func NewAcceptedWork(blockHash string, prevHash string, height uint32,
 }
 
 // fetchAcceptedWork fetches the accepted work referenced by the provided id.
+// Returns an error if the work is not found.
 func (db *BoltDB) fetchAcceptedWork(id string) (*AcceptedWork, error) {
 	const funcName = "fetchAcceptedWork"
 	var work AcceptedWork
@@ -122,7 +123,8 @@ func (db *BoltDB) persistAcceptedWork(work *AcceptedWork) error {
 	})
 }
 
-// updateAcceptedWork persists modifications to an existing work.
+// updateAcceptedWork persists modifications to an existing work. Returns an
+// error if the work is not found.
 func (db *BoltDB) updateAcceptedWork(work *AcceptedWork) error {
 	const funcName = "updateAcceptedWork"
 	return db.DB.Update(func(tx *bolt.Tx) error {

pool/boltdb.go

@@ -228,11 +228,9 @@ func nanoToBigEndianBytes(nano int64) []byte {
 	return b
 }
 
+// fetchPoolMode retrives the pool mode from the database. PoolMode is stored as
+// a uint32 for historical reasons. 0 indicates Public, 1 indicates Solo.
 func (db *BoltDB) fetchPoolMode() (uint32, error) {
-	// PoolMode is stored as a uint32 for historical reasons.
-	// 0 indicates Public
-	// 1 indicates Solo
-
 	var mode uint32
 	err := db.DB.View(func(tx *bolt.Tx) error {
 		pbkt, err := fetchPoolBucket(tx)
@@ -253,10 +251,10 @@ func (db *BoltDB) fetchPoolMode() (uint32, error) {
 	return mode, nil
 }
 
+// persistPoolMode stores the pool mode in the database. PoolMode is stored as a
+// uint32 for historical reasons. 0 indicates Public, 1 indicates Solo.
 func (db *BoltDB) persistPoolMode(mode uint32) error {
-	// PoolMode is stored as a uint32 for historical reasons.
-	// 0 indicates Public
-	// 1 indicates Solo
+
 	return db.DB.Update(func(tx *bolt.Tx) error {
 		pbkt := tx.Bucket(poolBkt)
 		b := make([]byte, 4)
@@ -265,6 +263,7 @@ func (db *BoltDB) persistPoolMode(mode uint32) error {
 	})
 }
 
+// fetchCSRFSecret retrieves the bytes used for the CSRF secret from the database.
 func (db *BoltDB) fetchCSRFSecret() ([]byte, error) {
 	var secret []byte
 
@@ -293,6 +292,7 @@ func (db *BoltDB) fetchCSRFSecret() ([]byte, error) {
 	return secret, nil
 }
 
+// persistCSRFSecret stores the bytes used for the CSRF secret in the database.
 func (db *BoltDB) persistCSRFSecret(secret []byte) error {
 	return db.DB.Update(func(tx *bolt.Tx) error {
 		pbkt := tx.Bucket(poolBkt)
@@ -305,6 +305,8 @@ func (db *BoltDB) persistCSRFSecret(secret []byte) error {
 	})
 }
 
+// persistLastPaymentInfo stores the last payment height and paidOn timestamp
+// in the database.
 func (db *BoltDB) persistLastPaymentInfo(height uint32, paidOn int64) error {
 	funcName := "persistLastPaymentInfo"
 	return db.DB.Update(func(tx *bolt.Tx) error {
@@ -333,6 +335,8 @@ func (db *BoltDB) persistLastPaymentInfo(height uint32, paidOn int64) error {
 	})
 }
 
+// loadLastPaymentInfo retrieves the last payment height and paidOn timestamp
+// from the database.
 func (db *BoltDB) loadLastPaymentInfo() (uint32, int64, error) {
 	funcName := "loadLastPaymentInfo"
 	var height uint32
@@ -364,6 +368,8 @@ func (db *BoltDB) loadLastPaymentInfo() (uint32, int64, error) {
 	return height, paidOn, nil
 }
 
+// persistLastPaymentCreatedOn stores the last payment createdOn timestamp in
+// the database.
 func (db *BoltDB) persistLastPaymentCreatedOn(createdOn int64) error {
 	funcName := "persistLastPaymentCreatedOn"
 	return db.DB.Update(func(tx *bolt.Tx) error {
@@ -381,6 +387,8 @@ func (db *BoltDB) persistLastPaymentCreatedOn(createdOn int64) error {
 	})
 }
 
+// loadLastPaymentCreatedOn retrieves the last payment createdOn timestamp from
+// the database.
 func (db *BoltDB) loadLastPaymentCreatedOn() (int64, error) {
 	funcName := "loadLastPaymentCreatedOn"
 	var createdOn int64
@@ -406,10 +414,13 @@ func (db *BoltDB) loadLastPaymentCreatedOn() (int64, error) {
 	return createdOn, nil
 }
 
+// Close closes the Bolt database.
 func (db *BoltDB) Close() error {
 	return db.DB.Close()
 }
 
+// httpBackup streams a backup of the entire database over the provided HTTP
+// response writer.
 func (db *BoltDB) httpBackup(w http.ResponseWriter) error {
 	err := db.DB.View(func(tx *bolt.Tx) error {
 		w.Header().Set("Content-Type", "application/octet-stream")