diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 19a57db9..2cd97d43 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,6 +7,21 @@ jobs: strategy: matrix: go: [1.14, 1.15] + + services: + postgres: + image: postgres:13.0 + env: + POSTGRES_USER: dcrpooluser + POSTGRES_PASSWORD: 12345 + POSTGRES_DB: dcrpooltestdb + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - name: Set up Go uses: actions/setup-go@v2 diff --git a/README.md b/README.md index fd80d819..3cdf5d99 100644 --- a/README.md +++ b/README.md @@ -41,17 +41,13 @@ of the address mining rewards are paid to and its name, formatted as: the address provided in the username to create an account, all other connected miners with the same address set will contribute work to that account. -As a contingency, the pool maintains a backup of the database (`backup.kv`), -created on shutdown in the same directory as the database itself. - The user interface of the pool provides public access to statistics and pool account data. Users of the pool can access all payments, mined blocks by the account and also work contributed by clients of the account via the interface. The interface is only accessible via HTTPS and by default uses a self-signed certificate, served on port `:8080`. In production, particularly for pool mining, a certificate from an authority (`CA`) like -[letsencrypt](https://letsencrypt.org/) is recommended. The user interface also -provides pool administrators database backup functionality when needed. +[letsencrypt](https://letsencrypt.org/) is recommended. ## Installing and Updating @@ -75,6 +71,17 @@ run `go install . ./cmd/...` in the root directory. Some notes: - The `dcrpool` executable will be installed to `$GOPATH/bin`. `GOPATH` defaults to `$HOME/go` (or `%USERPROFILE%\go` on Windows) if unset. +## Database + +dcrpool can run with either a [Bolt database](https://github.com/etcd-io/bbolt) +or a [Postgres database](https://www.postgresql.org/). Bolt is used by default. +[postgres.md](./docs/postgres.md) has more details about running with Postgres. + +When running in Bolt mode, the pool maintains a backup of the database +(`backup.kv`), created on shutdown in the same directory as the database itself. +The user interface also provides functionality for pool administrators to backup +Bolt database when necessary. + ### Example of obtaining and building from source on Ubuntu ```sh diff --git a/config.go b/config.go index dad64787..2bfe7a22 100644 --- a/config.go +++ b/config.go @@ -58,6 +58,12 @@ const ( defaultMaxConnectionsPerHost = 100 // 100 connected clients per host defaultWalletAccount = 0 defaultCoinbaseConfTimeout = time.Minute * 5 // one block time + defaultUsePostgres = false + defaultPGHost = "127.0.0.1" + defaultPGPort = 5432 + defaultPGUser = "dcrpooluser" + defaultPGPass = "12345" + defaultPGDBName = "dcrpooldb" ) var ( @@ -128,6 +134,12 @@ type config struct { DCR1Port uint32 `long:"dcr1port" ini-name:"dcr1port" description:"Obelisk DCR1 connection port."` CoinbaseConfTimeout time.Duration `long:"conftimeout" ini-name:"conftimeout" description:"The duration to wait for coinbase confirmations."` GenCertsOnly bool `long:"gencertsonly" ini-name:"gencertsonly" description:"Only generate needed TLS key pairs and terminate."` + UsePostgres bool `long:"postgres" ini-name:"postgres" description:"Use postgres database instead of bolt."` + PGHost string `long:"postgreshost" ini-name:"postgreshost" description:"Host to establish a postgres connection."` + PGPort uint32 `long:"postgresport" ini-name:"postgresport" description:"Port to establish a postgres connection."` + PGUser string `long:"postgresuser" ini-name:"postgresuser" description:"Username for postgres authentication."` + PGPass string `long:"postgrespass" ini-name:"postgrespass" description:"Password for postgres authentication."` + PGDBName string `long:"postgresdbname" ini-name:"postgresdbname" description:"Postgres database name."` poolFeeAddrs []dcrutil.Address dcrdRPCCerts []byte net *params @@ -362,6 +374,12 @@ func loadConfig() (*config, []string, error) { DCR1Port: defaultDCR1Port, WalletAccount: defaultWalletAccount, CoinbaseConfTimeout: defaultCoinbaseConfTimeout, + UsePostgres: defaultUsePostgres, + PGHost: defaultPGHost, + PGPort: defaultPGPort, + PGUser: defaultPGUser, + PGPass: defaultPGPass, + PGDBName: defaultPGDBName, } // Service options which are only added on Windows. diff --git a/dcrpool.go b/dcrpool.go index b6d94835..30c6c10c 100644 --- a/dcrpool.go +++ b/dcrpool.go @@ -212,13 +212,17 @@ func newPool(db pool.Database, cfg *config) (*miningPool, error) { FetchLastPaymentInfo: p.hub.FetchLastPaymentInfo, FetchMinedWork: p.hub.FetchMinedWork, FetchWorkQuotas: p.hub.FetchWorkQuotas, - HTTPBackupDB: p.hub.HTTPBackupDB, FetchClients: p.hub.FetchClients, AccountExists: p.hub.AccountExists, FetchArchivedPayments: p.hub.FetchArchivedPayments, FetchPendingPayments: p.hub.FetchPendingPayments, FetchCacheChannel: p.hub.FetchCacheChannel, } + + if !cfg.UsePostgres { + gcfg.HTTPBackupDB = p.hub.HTTPBackupDB + } + p.gui, err = gui.NewGUI(gcfg) if err != nil { p.hub.CloseListeners() @@ -245,7 +249,14 @@ func main() { } }() - db, err := pool.InitBoltDB(cfg.DBFile) + var db pool.Database + if cfg.UsePostgres { + db, err = pool.InitPostgresDB(cfg.PGHost, cfg.PGPort, cfg.PGUser, + cfg.PGPass, cfg.PGDBName) + } else { + db, err = pool.InitBoltDB(cfg.DBFile) + } + if err != nil { mpLog.Errorf("failed to initialize database: %v", err) os.Exit(1) @@ -292,11 +303,13 @@ func main() { p.hub.Run(p.ctx) // hub.Run() blocks until the pool is fully shut down. When it returns, - // write a backup of the DB, and then close the DB. - mpLog.Tracef("Backing up database.") - err = db.Backup(pool.BoltBackupFile) - if err != nil { - mpLog.Errorf("failed to write database backup file: %v", err) + // write a backup of the DB (if not using postgres), and then close the DB. + if !cfg.UsePostgres { + mpLog.Tracef("Backing up database.") + err = db.Backup(pool.BoltBackupFile) + if err != nil { + mpLog.Errorf("failed to write database backup file: %v", err) + } } db.Close() diff --git a/docs/postgres.md b/docs/postgres.md new file mode 100644 index 00000000..343e62c6 --- /dev/null +++ b/docs/postgres.md @@ -0,0 +1,49 @@ +# Running dcrpool with PostgreSQL + +Tested with PostgreSQL 13.0. + +**Note:** When running in Postgres mode, backups will not be created +automatically by dcrpool. + +## Setup + +1. Connect to your instance of PostgreSQL using `psql` to create a new database + and a new user for dcrpool. + Be sure to substitute the example password `12345` with something more secure. + + ```no-highlight + postgres=# CREATE DATABASE dcrpooldb; + CREATE DATABASE + postgres=# CREATE USER dcrpooluser WITH ENCRYPTED PASSWORD '12345'; + CREATE ROLE + postgres=# GRANT ALL PRIVILEGES ON DATABASE dcrpooldb to dcrpooluser; + GRANT + ``` + +1. **Developers only** - if you are modifying code and wish to run the dcrpool + test suite, you will need to create an additional database. + + ```no-highlight + postgres=# CREATE DATABASE dcrpooltestdb; + CREATE DATABASE + postgres=# GRANT ALL PRIVILEGES ON DATABASE dcrpooltestdb to dcrpooluser; + GRANT + ``` + +1. Add the database connection details to the dcrpool config file. + + ```no-highlight + postgres=true + postgreshost=127.0.0.1 + postgresport=5432 + postgresuser=dcrpooluser + postgrespass=12345 + postgresdbname=dcrpooldb + ``` + +## Tuning + +A helpful online tool to determine good settings for your system is called +[PGTune](https://pgtune.leopard.in.ua/#/). After providing basic information +about your hardware, PGTune will output a snippet of optimization settings to +add to your PostgreSQL config. diff --git a/go.mod b/go.mod index 8cf2be12..c2951b6b 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7 github.com/jrick/logrotate v1.0.0 github.com/kr/pretty v0.1.0 // indirect + github.com/lib/pq v1.8.0 go.etcd.io/bbolt v1.3.5 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 diff --git a/go.sum b/go.sum index e194e323..d17410fc 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= diff --git a/gui/admin.go b/gui/admin.go index 8e8f80f2..0c9b8efc 100644 --- a/gui/admin.go +++ b/gui/admin.go @@ -22,6 +22,7 @@ type adminPageData struct { ArchivedPayments []*archivedPayment PendingPaymentsTotal string PendingPayments []*pendingPayment + BackupAvailable bool } // adminPage is the handler for "GET /admin". If the current session is @@ -73,6 +74,7 @@ func (ui *GUI) adminPage(w http.ResponseWriter, r *http.Request) { PendingPayments: pendingPmts, ArchivedPaymentsTotal: totalArchived, ArchivedPayments: archivedPmts, + BackupAvailable: ui.cfg.HTTPBackupDB != nil, } ui.renderTemplate(w, "admin", pageData) diff --git a/gui/assets/templates/admin.html b/gui/assets/templates/admin.html index 53e90f6b..e57ee1a6 100644 --- a/gui/assets/templates/admin.html +++ b/gui/assets/templates/admin.html @@ -9,11 +9,12 @@

Admin Panel

+ {{ if .BackupAvailable }}
{{.HeaderData.CSRF}}
- + {{ end }}
{{.HeaderData.CSRF}} diff --git a/harness.sh b/harness.sh index ea3f5ff1..b94d520d 100755 --- a/harness.sh +++ b/harness.sh @@ -29,6 +29,15 @@ MINER_MAX_PROCS=1 PAYMENT_METHOD="pplns" LAST_N_PERIOD=5m GUI_DIR="${HARNESS_ROOT}/gui" + +# Using postgres requires the DB specified below to exist and contain no data. +USE_POSTGRES=true +POSTGRES_HOST=127.0.0.1 +POSTGRES_PORT=5432 +POSTGRES_USER=dcrpooluser +POSTGRES_PASS=12345 +POSTGRES_DBNAME=dcrpooldb + # CPU_MINING_ADDR is the mining address printed during creation of vwallet. # Initial block rewards from `generate` are sent here so vwallet can buy tickets. CPU_MINING_ADDR="SsaJxXSymEGroxAiUY9u1mRq1DDWLxn5WhB" @@ -128,6 +137,12 @@ adminpass=${ADMIN_PASS} guidir=${GUI_DIR} designation=${TMUX_SESSION} profile=6060 +postgres=${USE_POSTGRES} +postgreshost=${POSTGRES_HOST} +postgresport=${POSTGRES_PORT} +postgresuser=${POSTGRES_USER} +postgrespass=${POSTGRES_PASS} +postgresdbname=${POSTGRES_DBNAME} EOF cat > "${HARNESS_ROOT}/mwallet/dcrmwctl.conf" <(estimatedmaturity+1);` + + selectPendingPayments = ` + SELECT + uuid, + account, + estimatedmaturity, + height, + amount, + createdon, + paidonheight, + transactionid, + sourceblockhash, + sourcecoinbase + FROM payments + WHERE paidonheight=0;` + + countPaymentsAtBlockHash = ` + SELECT count(1) + FROM payments + WHERE paidonheight=0 + AND sourceblockhash=$1;` + + selectArchivedPayments = ` + SELECT + uuid, + account, + estimatedmaturity, + height, + amount, + createdon, + paidonheight, + transactionid, + sourceblockhash, + sourcecoinbase + FROM archivedpayments + ORDER BY height DESC;` + + selectMaturePendingPayments = ` + SELECT + uuid, + account, + estimatedmaturity, + height, + amount, + createdon, + paidonheight, + transactionid, + sourceblockhash, + sourcecoinbase + FROM payments + WHERE paidonheight=0 + AND (estimatedmaturity+1)<=$1;` + + selectShare = ` + SELECT + uuid, account, weight, createdon + FROM shares + WHERE uuid=$1;` + + insertShare = ` + INSERT INTO shares( + uuid, account, weight, createdon + ) + VALUES ($1,$2,$3,$4);` + + selectSharesOnOrBeforeTime = ` + SELECT + uuid, account, weight, createdon + FROM shares + WHERE createdon <= $1` + + selectSharesAfterTime = ` + SELECT + uuid, account, weight, createdon + FROM shares + WHERE createdon > $1` + + deleteShareCreatedBefore = `DELETE FROM shares WHERE createdon < $1` + + selectAcceptedWork = ` + SELECT + uuid, + blockhash, + prevhash, + height, + minedby, + miner, + createdon, + confirmed + FROM acceptedwork + WHERE uuid=$1;` + + insertAcceptedWork = ` + INSERT INTO acceptedwork( + uuid, + blockhash, + prevhash, + height, + minedby, + miner, + createdon, + confirmed + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8);` + + updateAcceptedWork = ` + UPDATE acceptedwork + SET + blockhash=$2, + prevhash=$3, + height=$4, + minedby=$5, + miner=$6, + createdon=$7, + confirmed=$8 + WHERE uuid=$1;` + + deleteAcceptedWork = `DELETE FROM acceptedwork WHERE uuid=$1;` + + selectMinedWork = ` + SELECT + uuid, + blockhash, + prevhash, + height, + minedby, + miner, + createdon, + confirmed + FROM acceptedwork + ORDER BY height DESC;` + + selectUnconfirmedWork = ` + SELECT + uuid, + blockhash, + prevhash, + height, + minedby, + miner, + createdon, + confirmed + FROM acceptedwork + WHERE $1>height + AND confirmed=false;` + + selectJob = `SELECT uuid, header, height FROM jobs WHERE uuid=$1;` + + insertJob = `INSERT INTO jobs(uuid, height, header) VALUES ($1,$2,$3);` + + deleteJob = `DELETE FROM jobs WHERE uuid=$1;` + + deleteJobBeforeHeight = `DELETE FROM jobs WHERE height < $1;` +)