Skip to content
This repository has been archived by the owner on Apr 12, 2023. It is now read-only.

Commit

Permalink
treemanager: Expose read-only mode and fast reads
Browse files Browse the repository at this point in the history
This exposes the driver's read-only mode and fast reads options to the
tree manager server. When used, this will run the tree manager in a mode
that disallows any write operations. If fast reads are used, it will
attempt to use the fast read or inconsistent mode. In CockroachDB this
translates to using follower reads to allow an instance to fetch data
from a follower instead of relying on the leader.

Signed-off-by: Juan Antonio Osorio <juan.osoriorobles@eu.equinix.com>
  • Loading branch information
JAORMX committed Dec 23, 2022
1 parent fd49647 commit c5f6b09
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 33 deletions.
42 changes: 28 additions & 14 deletions cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"net/http"
"os"
"os/signal"
"time"

"github.com/spf13/cobra"
"github.com/spf13/viper"
Expand All @@ -24,11 +23,6 @@ import (
dbutils "github.com/infratographer/fertilesoil/storage/crdb/utils"
)

const (
defaultListen = ":8080"
defaultServerShutdownTimeout = 5 * time.Second
)

// serveCmd represents the treemanager command.
var serveCmd = &cobra.Command{
Use: "serve",
Expand All @@ -48,18 +42,36 @@ func init() {

v := viper.GetViper()
dbutils.RegisterDBArgs(v, serveCmd.Flags())
ginx.MustViperFlags(v, serveCmd.Flags(), defaultListen)
ginx.MustViperFlags(v, serveCmd.Flags(), treemanager.DefaultTreeManagerListen)
loggingx.MustViperFlags(v, serveCmd.Flags())
natsutils.RegisterNATSArgs(v, serveCmd.Flags())

// TODO(jaosorior): Add tracing
// TODO(jaosorior): Add metrics
// TODO(jaosorior): Add TLS flags

// Server flags
flags := serveCmd.Flags()

// server shutdown timeout
flags.Duration("server-shutdown-timeout",
defaultServerShutdownTimeout,
treemanager.DefaultTreeManagerShutdownTimeout,
"Time to wait for the server to shutdown gracefully")
viperx.MustBindFlag(v, "server.shutdown", flags.Lookup("server-shutdown-timeout"))
flags.String("server-unix-socket", "", "Listen on a unix socket instead of a TCP socket.")

// server UNIX socket
flags.String("server-unix-socket",
treemanager.DefaultTreeManagerUnix,
"Listen on a unix socket instead of a TCP socket.")
viperx.MustBindFlag(v, "server.unix_socket", flags.Lookup("server-unix-socket"))

// read-only mode
flags.Bool("read-only", treemanager.DefaultTreeManagerReadOnly, "Run the server in read-only mode.")
viperx.MustBindFlag(v, "server.read_only", flags.Lookup("read-only"))

// fast reads
flags.Bool("fast-reads", treemanager.DefaultTreeManagerFastReads, "Run the server in fast reads mode.")
viperx.MustBindFlag(v, "server.fast_reads", flags.Lookup("fast-reads"))
}

func serverRunE(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -99,12 +111,14 @@ func serverRunE(cmd *cobra.Command, args []string) error {

s := treemanager.NewServer(
l,
viper.GetString("listen"),
db,
viper.GetBool("debug"),
viper.GetDuration("server.shutdown"),
viper.GetString("server.unix_socket"),
notif,
treemanager.WithListen(v.GetString("server.listen")),
treemanager.WithUnix(v.GetString("server.unix_socket")),
treemanager.WithDebug(v.GetBool("debug")),
treemanager.WithReadOnly(v.GetBool("server.read_only")),
treemanager.WithFastReads(v.GetBool("server.fast_reads")),
treemanager.WithShutdownTimeout(v.GetDuration("server.shutdown")),
treemanager.WithNotifier(notif),
)

go func() {
Expand Down
100 changes: 100 additions & 0 deletions internal/httpsrv/treemanager/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package treemanager

import (
"time"

"github.com/infratographer/fertilesoil/notifier"
"github.com/infratographer/fertilesoil/notifier/noop"
"github.com/infratographer/fertilesoil/storage/crdb/driver"
)

type treeManagerConfig struct {
listen string
unix string
debug bool
readonly bool
fastReads bool
shutdownTimeout time.Duration
notif notifier.Notifier
}

type Option func(*treeManagerConfig)

// WithListen sets the listen address for the server.
func WithListen(listen string) Option {
return func(c *treeManagerConfig) {
c.listen = listen
}
}

// WithUnix sets the unix socket for the server.
// If set, the server will listen on the unix socket
// instead of the listen address.
func WithUnix(unix string) Option {
return func(c *treeManagerConfig) {
c.unix = unix
}
}

// WithDebug sets the debug flag for the server.
func WithDebug(debug bool) Option {
return func(c *treeManagerConfig) {
c.debug = debug
}
}

// WithReadOnly sets the readonly flag for the server.
// If true, the server will use the readonly flag when
// reading from the database.
func WithReadOnly(readonly bool) Option {
return func(c *treeManagerConfig) {
c.readonly = readonly
}
}

// WithFastReads sets the fastReads flag for the server.
// If true, the server will use the fastReads flag when
// reading from the database. This will cause the server
// to read from the database without waiting for the
// database to commit the transaction. This is useful
// for read-heavy workloads, but can cause data to be
// out of sync with the database.
func WithFastReads(fastReads bool) Option {
return func(c *treeManagerConfig) {
c.fastReads = fastReads
}
}

// WithShutdownTimeout sets the shutdown timeout for the server.
func WithShutdownTimeout(t time.Duration) Option {
return func(c *treeManagerConfig) {
c.shutdownTimeout = t
}
}

// WithNotifier sets the notifier for the server.
func WithNotifier(n notifier.Notifier) Option {
return func(c *treeManagerConfig) {
if n == nil {
n = noop.NewNotifier()
}
c.notif = n
}
}

func (c *treeManagerConfig) apply(opts ...Option) {
for _, opt := range opts {
opt(c)
}
}

func (c *treeManagerConfig) withStorageDriverOptions() []driver.Options {
opts := []driver.Options{}
if c.readonly {
opts = append(opts, driver.WithReadOnly())
}
if c.fastReads {
opts = append(opts, driver.WithFastReads())
}
return opts
}
27 changes: 27 additions & 0 deletions internal/httpsrv/treemanager/defaults.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package treemanager

import (
"time"

"github.com/infratographer/fertilesoil/notifier/noop"
)

const (
// DefaultTreeManagerListen is the default listen address for the TreeManager.
DefaultTreeManagerListen = ":8080"
// DefaultTreeManagerUnix is the default unix socket for the TreeManager.
DefaultTreeManagerUnix = ""
// DefaultTreeManagerDebug is the default debug flag for the TreeManager.
DefaultTreeManagerDebug = false
// DefaultTreeManagerReadOnly is the default read-only flag for the TreeManager.
DefaultTreeManagerReadOnly = false
// DefaultTreeManagerFastReads is the default fast reads flag for the TreeManager.
DefaultTreeManagerFastReads = false
// DefaultTreeManagerShutdownTimeout is the default shutdown timeout for the TreeManager.
DefaultTreeManagerShutdownTimeout = 5 * time.Second
)

var (
// DefaultTreeManagerNotifier is the default notifier for the TreeManager.
DefaultTreeManagerNotifier = noop.NewNotifier()
)
31 changes: 14 additions & 17 deletions internal/httpsrv/treemanager/treemanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,38 @@ import (
"database/sql"
"errors"
"net/http"
"time"

"github.com/gin-gonic/gin"
"go.uber.org/zap"

v1 "github.com/infratographer/fertilesoil/api/v1"
"github.com/infratographer/fertilesoil/internal/httpsrv/common"
"github.com/infratographer/fertilesoil/notifier"
"github.com/infratographer/fertilesoil/notifier/noop"
"github.com/infratographer/fertilesoil/storage"
"github.com/infratographer/fertilesoil/storage/crdb/driver"
sn "github.com/infratographer/fertilesoil/storage/notifier"
)

func NewServer(
logger *zap.Logger,
listen string,
db *sql.DB,
debug bool,
shutdownTime time.Duration,
unix string,
n notifier.Notifier,
opts ...Option,
) *common.Server {
dbdrv := driver.NewDirectoryDriver(db)
cfg := &treeManagerConfig{
listen: DefaultTreeManagerListen,
unix: DefaultTreeManagerUnix,
debug: DefaultTreeManagerDebug,
readonly: DefaultTreeManagerReadOnly,
fastReads: DefaultTreeManagerFastReads,
shutdownTimeout: DefaultTreeManagerShutdownTimeout,
notif: DefaultTreeManagerNotifier,
}
cfg.apply(opts...)

var notif notifier.Notifier
dbdrv := driver.NewDirectoryDriver(db, cfg.withStorageDriverOptions()...)

if n == nil {
notif = noop.NewNotifier()
} else {
notif = n
}
store := sn.StorageWithNotifier(dbdrv, cfg.notif, sn.WithNotifyRetrier())

store := sn.StorageWithNotifier(dbdrv, notif, sn.WithNotifyRetrier())
s := common.NewServer(logger, listen, db, store, debug, shutdownTime, unix)
s := common.NewServer(logger, cfg.listen, db, store, cfg.debug, cfg.shutdownTimeout, cfg.unix)

s.SetHandler(newHandler(logger, s))

Expand Down
9 changes: 8 additions & 1 deletion tests/integration/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,14 @@ func TestServerWithBadDB(t *testing.T) {
dbconn, err := sql.Open("postgres", baseDBURL.String())
assert.NoError(t, err, "error creating db connection")

srv := treemanager.NewServer(tl, srvhost, dbconn, debug, defaultShutdownTime, skt, nil)
srv := treemanager.NewServer(
tl,
dbconn,
treemanager.WithListen(srvhost),
treemanager.WithDebug(debug),
treemanager.WithShutdownTimeout(defaultShutdownTime),
treemanager.WithUnix(skt),
)

defer func() {
err := srv.Shutdown()
Expand Down
10 changes: 9 additions & 1 deletion tests/integration/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,15 @@ func newTestServerWithNotifier(t *testing.T, skt string, notif notifier.Notifier

gooseDBMutex.Unlock()

tm := treemanager.NewServer(tl, srvhost, dbconn, debug, defaultShutdownTime, skt, notif)
tm := treemanager.NewServer(
tl,
dbconn,
treemanager.WithListen(srvhost),
treemanager.WithDebug(debug),
treemanager.WithShutdownTimeout(defaultShutdownTime),
treemanager.WithUnix(skt),
treemanager.WithNotifier(notif),
)

return tm
}
Expand Down

0 comments on commit c5f6b09

Please sign in to comment.