SQL Storage Support (#37)

* First pass at sql/postgresql support

* didn't realize go build adds but never removes from go.md, delete the sqlx deps

* Migrate storage to uri style

add in sqlite, postgres, mysql

Tests don't pass cause they don't have a db to talk to

Signed-off-by: Gavin Mogan <git@gavinmogan.com>

* update docs

* Refactor to not open till main.go, makes tests work, easier to default to inmemory

.gitignore

@@ -45,6 +45,7 @@ website/.pnp.js
 
 # testing
 website/coverage
+internal/storage/sqlite.db
 
 # production
 website/build
@@ -62,4 +63,3 @@ website/yarn-debug.log*
 website/yarn-error.log*
 website/yarn.lock
 
-

docs/configuration.md

@@ -41,15 +41,9 @@ disableMetadata: false
 # The port that the web ui server (http) will listen on.
 # Optional, defaults to 8000
 port: 8000
-storage:
-  # Directory that VPN devices (WireGuard peers)
-  # should be saved under.
-  # If this value is empty then an InMemory storage
-  # backend will be used (not recommended).
-  # Optional
-  # Defaults to in-memory
-  # The docker container sets this value to /data automatically
-  directory: /data
+# Directory that VPN devices (WireGuard peers)
+# What type of storage do you want? inmemory (default), disk:///path/to/thing, or postgresql, mysql, sqlite3
+storage: ""
 wireguard:
   # The network interface name for wireguard
   # Optional, defaults to wg0

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
 	github.com/improbable-eng/grpc-web v0.12.0
 	github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 // indirect
+	github.com/jinzhu/gorm v1.9.12
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/miekg/dns v1.1.27
@@ -28,6 +29,7 @@ require (
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/stretchr/testify v1.4.0
 	github.com/tg123/go-htpasswd v1.0.0
 	github.com/vishvananda/netlink v1.0.0
 	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413

go.sum

@@ -16,6 +16,7 @@ github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I=
 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE=
 github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
@@ -24,11 +25,15 @@ github.com/docker/libnetwork v0.8.0-dev.2.0.20200217033114-6659f7f4d8c1 h1:Y1inp
 github.com/docker/libnetwork v0.8.0-dev.2.0.20200217033114-6659f7f4d8c1/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -58,6 +63,11 @@ github.com/improbable-eng/grpc-web v0.12.0 h1:GlCS+lMZzIkfouf7CNqY+qqpowdKuJLSLL
 github.com/improbable-eng/grpc-web v0.12.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs=
 github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 h1:rw3IAne6CDuVFlZbPOkA7bhxlqawFh7RJJ+CejfMaxE=
 github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
+github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q=
+github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a h1:84IpUNXj4mCR9CuCEvSiCArMbzr/TMbuPIadKDwypkI=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
@@ -74,6 +84,10 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
+github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mdlayher/genetlink v0.0.0-20191205172946-651acf4b47ef h1:VOblll+3pOfnsJfEjrEX3TeKeF/gKkXOK20KMR7II+8=
 github.com/mdlayher/genetlink v0.0.0-20191205172946-651acf4b47ef/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
 github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
@@ -123,12 +137,14 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191205161847-0a08dada0ff9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

internal/config/config.go

@@ -11,6 +11,7 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/place1/wg-access-server/internal/network"
+	"github.com/place1/wg-access-server/internal/storage"
 	"github.com/place1/wg-access-server/pkg/authnz/authconfig"
 	"github.com/vishvananda/netlink"
 
@@ -28,14 +29,8 @@ type AppConfig struct {
 	AdminPassword   string `yaml:"adminPassword"`
 	// Port sets the port that the web UI will listen on.
 	// Defaults to 8000
-	Port    int `yaml:"port"`
-	Storage struct {
-		// Directory that VPN devices (WireGuard peers)
-		// should be saved under.
-		// If this value is empty then an InMemory storage
-		// backend will be used (not recommended).
-		Directory string `yaml:"directory"`
-	} `yaml:"storage"`
+	Port      int                    `yaml:"port"`
+	Storage   storage.StorageWrapper `yaml:"storage"`
 	WireGuard struct {
 		// The network interface name of the WireGuard
 		// network device.
@@ -117,7 +112,11 @@ func Read() *AppConfig {
 	config.WireGuard.Port = *wireguardPort
 	config.VPN.CIDR = "10.44.0.0/24"
 	config.DisableMetadata = *disableMetadata
-	config.Storage.Directory = *storagePath
+	s, err := storage.NewStorageWrapper(*storagePath)
+	if err != nil {
+		logrus.Fatal(errors.Wrap(err, "failed to bind configuration file"))
+	}
+	config.Storage = *s
 	config.WireGuard.PrivateKey = *privateKey
 
 	if config.DNS.Enabled == nil {
@@ -188,16 +187,6 @@ func Read() *AppConfig {
 		config.WireGuard.PrivateKey = key.String()
 	}
 
-	if config.Storage.Directory == "" {
-		logrus.Warn("storage directory not configured - using in-memory storage backend! wireguard devices will be lost when the process exits!")
-	} else {
-		config.Storage.Directory, err = filepath.Abs(config.Storage.Directory)
-		if err != nil {
-			logrus.Fatal(errors.Wrap(err, "failed to get absolute path to storage directory"))
-		}
-		os.MkdirAll(config.Storage.Directory, 0700)
-	}
-
 	if config.AdminPassword != "" {
 		if config.Auth.Basic == nil {
 			config.Auth.Basic = &authconfig.BasicAuthConfig{}

internal/devices/devices.go

@@ -3,8 +3,6 @@ package devices
 import (
 	"fmt"
 	"net"
-	"os"
-	"path/filepath"
 	"sync"
 	"time"
 
@@ -79,27 +77,23 @@ func (d *DeviceManager) AddDevice(identity *authsession.Identity, name string, p
 }
 
 func (d *DeviceManager) SaveDevice(device *storage.Device) error {
-	return d.storage.Save(key(device.Owner, device.Name), device)
+	return d.storage.Save(device)
 }
 
 func (d *DeviceManager) ListAllDevices() ([]*storage.Device, error) {
 	return d.storage.List("")
 }
 
 func (d *DeviceManager) ListDevices(user string) ([]*storage.Device, error) {
-	prefix := ""
-	if user != "" {
-		prefix = user + string(os.PathSeparator)
-	}
-	return d.storage.List(prefix)
+	return d.storage.List(user)
 }
 
 func (d *DeviceManager) DeleteDevice(user string, name string) error {
-	device, err := d.storage.Get(key(user, name))
+	device, err := d.storage.Get(user, name)
 	if err != nil {
 		return errors.Wrap(err, "failed to retrieve device")
 	}
-	if err := d.storage.Delete(key(user, name)); err != nil {
+	if err := d.storage.Delete(device); err != nil {
 		return err
 	}
 	if err := wgembed.RemovePeer(d.iface, device.PublicKey); err != nil {
@@ -108,10 +102,6 @@ func (d *DeviceManager) DeleteDevice(user string, name string) error {
 	return nil
 }
 
-func key(user string, device string) string {
-	return filepath.Join(user, device)
-}
-
 var nextIPLock = sync.Mutex{}
 
 func (d *DeviceManager) nextClientAddress() (string, error) {

internal/storage/contracts.go

@@ -5,21 +5,23 @@ import (
 )
 
 type Storage interface {
-	Save(key string, device *Device) error
-	List(prefix string) ([]*Device, error)
-	Get(key string) (*Device, error)
-	Delete(key string) error
+	Save(device *Device) error
+	List(owner string) ([]*Device, error)
+	Get(owner string, name string) (*Device, error)
+	Delete(device *Device) error
+	Close() error
+	Open() error
 }
 
 type Device struct {
-	Owner         string    `json:"owner"`
+	Owner         string    `json:"owner" gorm:"type:varchar(100);unique_index:key"`
 	OwnerName     string    `json:"ownerName"`
 	OwnerEmail    string    `json:"ownerEmail"`
 	OwnerProvider string    `json:"ownerProvider"`
-	Name          string    `json:"name"`
+	Name          string    `json:"name" gorm:"type:varchar(100);unique_index:key"`
 	PublicKey     string    `json:"publicKey"`
 	Address       string    `json:"address"`
-	CreatedAt     time.Time `json:"createdAt"`
+	CreatedAt     time.Time `json:"createdAt" gorm:"column:created_at"`
 
 	/**
 	 * Metadata fields below.

internal/storage/disk.go

@@ -18,15 +18,24 @@ type DiskStorage struct {
 }
 
 func NewDiskStorage(directory string) *DiskStorage {
-	if _, err := os.Stat(directory); os.IsNotExist(err) {
-		if err := os.MkdirAll(directory, 0600); err != nil {
-			logrus.Fatal(errors.Wrap(err, "failed to create storage directory"))
+	return &DiskStorage{directory}
+}
+
+func (s *DiskStorage) Open() error {
+	if _, err := os.Stat(s.directory); os.IsNotExist(err) {
+		if err := os.MkdirAll(s.directory, 0600); err != nil {
+			return errors.Wrap(err, "failed to create storage directory")
 		}
 	}
-	return &DiskStorage{directory}
+	return nil
 }
 
-func (s *DiskStorage) Save(key string, device *Device) error {
+func (s *DiskStorage) Close() error {
+	return nil
+}
+
+func (s *DiskStorage) Save(device *Device) error {
+	key := key(device)
 	path := s.deviceFilePath(key)
 	logrus.Debugf("saving device %s", path)
 	bytes, err := json.Marshal(device)
@@ -40,7 +49,13 @@ func (s *DiskStorage) Save(key string, device *Device) error {
 	return nil
 }
 
-func (s *DiskStorage) List(prefix string) ([]*Device, error) {
+func (s *DiskStorage) List(username string) ([]*Device, error) {
+	prefix := func() string {
+		if username != "" {
+			return keyStr(username, "")
+		}
+		return ""
+	}()
 	files := []string{}
 	err := filepath.Walk(s.directory, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -80,7 +95,8 @@ func (s *DiskStorage) List(prefix string) ([]*Device, error) {
 	return devices, nil
 }
 
-func (s *DiskStorage) Get(key string) (*Device, error) {
+func (s *DiskStorage) Get(owner string, name string) (*Device, error) {
+	key := keyStr(owner, name)
 	path := s.deviceFilePath(key)
 	bytes, err := ioutil.ReadFile(path)
 	if err != nil {
@@ -93,8 +109,8 @@ func (s *DiskStorage) Get(key string) (*Device, error) {
 	return device, nil
 }
 
-func (s *DiskStorage) Delete(key string) error {
-	if err := os.Remove(s.deviceFilePath(key)); err != nil {
+func (s *DiskStorage) Delete(device *Device) error {
+	if err := os.Remove(s.deviceFilePath(key(device))); err != nil {
 		return errors.Wrap(err, "failed to delete device file")
 	}
 	return nil

internal/storage/inmemory.go

@@ -5,39 +5,56 @@ import (
 	"strings"
 )
 
-var memory = map[string]*Device{}
-
 // implements Storage interface
-type InMemoryStorage struct{}
+type InMemoryStorage struct {
+	db map[string]*Device
+}
 
 func NewMemoryStorage() *InMemoryStorage {
-	return &InMemoryStorage{}
+	db := make(map[string]*Device)
+	return &InMemoryStorage{
+		db: db,
+	}
+}
+
+func (s *InMemoryStorage) Open() error {
+	return nil
 }
 
-func (s *InMemoryStorage) Save(key string, device *Device) error {
-	memory[key] = device
+func (s *InMemoryStorage) Close() error {
 	return nil
 }
 
-func (s *InMemoryStorage) List(prefix string) ([]*Device, error) {
+func (s *InMemoryStorage) Save(device *Device) error {
+	s.db[key(device)] = device
+	return nil
+}
+
+func (s *InMemoryStorage) List(username string) ([]*Device, error) {
 	devices := []*Device{}
-	for key, device := range memory {
+	prefix := func() string {
+		if username != "" {
+			return keyStr(username, "")
+		}
+		return ""
+	}()
+	for key, device := range s.db {
 		if strings.HasPrefix(key, prefix) {
 			devices = append(devices, device)
 		}
 	}
 	return devices, nil
 }
 
-func (s *InMemoryStorage) Get(key string) (*Device, error) {
-	device, ok := memory[key]
+func (s *InMemoryStorage) Get(owner string, name string) (*Device, error) {
+	device, ok := s.db[keyStr(owner, name)]
 	if !ok {
 		return nil, errors.New("device doesn't exist")
 	}
 	return device, nil
 }
 
-func (s *InMemoryStorage) Delete(key string) error {
-	delete(memory, key)
+func (s *InMemoryStorage) Delete(device *Device) error {
+	delete(s.db, key(device))
 	return nil
 }