From 21f0a344def526aab82311f5c08d109183390d3a Mon Sep 17 00:00:00 2001 From: Stefan Richter Date: Sun, 20 Nov 2022 21:47:19 -0800 Subject: [PATCH] Adding support for PostgreSQL as database This adds support for a second database backend: PostgreSQL (in addition to sqlite3). This allows externailzing the database used by gonic. --- cmd/gonic/gonic.go | 24 ++-- db/db.go | 106 ++++++------------ db/db_test.go | 2 +- db/migrations.go | 52 +++++---- mockfs/mockfs.go | 20 +--- scanner/scanner_fuzz_test.go | 1 - scanner/scanner_test.go | 1 - server/ctrlsubsonic/handlers_by_folder.go | 6 +- .../ctrlsubsonic/handlers_by_folder_test.go | 2 - server/ctrlsubsonic/handlers_by_tags.go | 6 +- server/ctrlsubsonic/handlers_raw.go | 2 +- 11 files changed, 88 insertions(+), 134 deletions(-) diff --git a/cmd/gonic/gonic.go b/cmd/gonic/gonic.go index 625c348e..f6ac8486 100644 --- a/cmd/gonic/gonic.go +++ b/cmd/gonic/gonic.go @@ -3,6 +3,7 @@ package main import ( "context" + "encoding/base64" "errors" "expvar" "flag" @@ -26,6 +27,7 @@ import ( "github.com/google/shlex" "github.com/gorilla/securecookie" + _ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/sentriz/gormstore" "golang.org/x/sync/errgroup" @@ -65,8 +67,7 @@ func main() { flag.Var(&confMusicPaths, "music-path", "path to music") confPlaylistsPath := flag.String("playlists-path", "", "path to your list of new or existing m3u playlists that gonic can manage") - - confDBPath := flag.String("db-path", "gonic.db", "path to database (optional)") + confDBURI := flag.String("db-uri", "", "db URI") confScanIntervalMins := flag.Uint("scan-interval", 0, "interval (in minutes) to automatically scan music (optional)") confScanAtStart := flag.Bool("scan-at-start-enabled", false, "whether to perform an initial scan at startup (optional)") @@ -92,6 +93,7 @@ func main() { confExpvar := flag.Bool("expvar", false, "enable the /debug/vars endpoint (optional)") deprecatedConfGenreSplit := flag.String("genre-split", "", "(deprecated, see multi-value settings)") + deprecatedConfDBPath := flag.String("db-path", "gonic.db", "(deprecated, see db-uri)") flag.Parse() flagconf.ParseEnv() @@ -136,7 +138,11 @@ func main() { log.Fatalf("couldn't create covers cache path: %v\n", err) } - dbc, err := db.New(*confDBPath, db.DefaultOptions()) + if *confDBURI == "" { + *confDBURI = "sqlite3://" + *deprecatedConfDBPath + } + + dbc, err := db.New(*confDBURI) if err != nil { log.Fatalf("error opening database: %v\n", err) } @@ -144,7 +150,6 @@ func main() { err = dbc.Migrate(db.MigrationContext{ Production: true, - DBPath: *confDBPath, OriginalMusicPath: confMusicPaths[0].path, PlaylistsPath: *confPlaylistsPath, PodcastsPath: *confPodcastPath, @@ -225,17 +230,18 @@ func main() { jukebx = jukebox.New() } - sessKey, err := dbc.GetSetting("session_key") + encSessKey, err := dbc.GetSetting("session_key") if err != nil { log.Panicf("error getting session key: %v\n", err) } - if sessKey == "" { - sessKey = string(securecookie.GenerateRandomKey(32)) - if err := dbc.SetSetting("session_key", sessKey); err != nil { + sessKey, err := base64.StdEncoding.DecodeString(encSessKey) + if err != nil || len(sessKey) == 0 { + sessKey = securecookie.GenerateRandomKey(32) + if err := dbc.SetSetting("session_key", base64.StdEncoding.EncodeToString(sessKey)); err != nil { log.Panicf("error setting session key: %v\n", err) } } - sessDB := gormstore.New(dbc.DB, []byte(sessKey)) + sessDB := gormstore.New(dbc.DB, []byte(encSessKey)) sessDB.SessionOpts.HttpOnly = true sessDB.SessionOpts.SameSite = http.SameSiteLaxMode diff --git a/db/db.go b/db/db.go index 9ea81a97..d910dbbb 100644 --- a/db/db.go +++ b/db/db.go @@ -1,7 +1,6 @@ package db import ( - "context" "errors" "fmt" "log" @@ -13,55 +12,55 @@ import ( "time" "github.com/jinzhu/gorm" - "github.com/mattn/go-sqlite3" + _ "github.com/jinzhu/gorm/dialects/postgres" + _ "github.com/jinzhu/gorm/dialects/sqlite" // TODO: remove this dep "go.senan.xyz/gonic/server/ctrlsubsonic/specid" ) -func DefaultOptions() url.Values { - return url.Values{ - // with this, multiple connections share a single data and schema cache. - // see https://www.sqlite.org/sharedcache.html - "cache": {"shared"}, - // with this, the db sleeps for a little while when locked. can prevent - // a SQLITE_BUSY. see https://www.sqlite.org/c3ref/busy_timeout.html - "_busy_timeout": {"30000"}, - "_journal_mode": {"WAL"}, - "_foreign_keys": {"true"}, - } +type DB struct { + *gorm.DB } -func mockOptions() url.Values { - return url.Values{ - "_foreign_keys": {"true"}, +func New(uri string) (*DB, error) { + if uri == "" { + return nil, fmt.Errorf("empty db uri") } -} -type DB struct { - *gorm.DB -} + url, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("parse uri: %w", err) + } -func New(path string, options url.Values) (*DB, error) { - // https://github.com/mattn/go-sqlite3#connection-string - url := url.URL{ - Scheme: "file", - Opaque: path, + gormURL := strings.TrimPrefix(url.String(), url.Scheme+"://") + + //nolint:goconst + switch url.Scheme { + case "sqlite3": + q := url.Query() + q.Set("cache", "shared") + q.Set("_busy_timeout", "30000") + q.Set("_journal_mode", "WAL") + q.Set("_foreign_keys", "true") + url.RawQuery = q.Encode() + case "postgres": + // the postgres driver expects the schema prefix to be on the URL + gormURL = url.String() + default: + return nil, fmt.Errorf("unknown db scheme") } - url.RawQuery = options.Encode() - db, err := gorm.Open("sqlite3", url.String()) + + db, err := gorm.Open(url.Scheme, gormURL) if err != nil { return nil, fmt.Errorf("with gorm: %w", err) } + db.SetLogger(log.New(os.Stdout, "gorm ", 0)) db.DB().SetMaxOpenConns(1) return &DB{DB: db}, nil } -func NewMock() (*DB, error) { - return New(":memory:", mockOptions()) -} - func (db *DB) InsertBulkLeftMany(table string, head []string, left int, col []int) error { if len(col) == 0 { return nil @@ -72,10 +71,11 @@ func (db *DB) InsertBulkLeftMany(table string, head []string, left int, col []in rows = append(rows, "(?, ?)") values = append(values, left, c) } - q := fmt.Sprintf("INSERT OR IGNORE INTO %q (%s) VALUES %s", + q := fmt.Sprintf("INSERT INTO %q (%s) VALUES %s ON CONFLICT (%s) DO NOTHING", table, strings.Join(head, ", "), strings.Join(rows, ", "), + strings.Join(head, ", "), ) return db.Exec(q, values...).Error } @@ -611,45 +611,3 @@ func join[T fmt.Stringer](in []T, sep string) string { } return strings.Join(strs, sep) } - -func Dump(ctx context.Context, db *gorm.DB, to string) error { - dest, err := New(to, url.Values{}) - if err != nil { - return fmt.Errorf("create dest db: %w", err) - } - defer dest.Close() - - connSrc, err := db.DB().Conn(ctx) - if err != nil { - return fmt.Errorf("getting src raw conn: %w", err) - } - defer connSrc.Close() - - connDest, err := dest.DB.DB().Conn(ctx) - if err != nil { - return fmt.Errorf("getting dest raw conn: %w", err) - } - defer connDest.Close() - - err = connDest.Raw(func(connDest interface{}) error { - return connSrc.Raw(func(connSrc interface{}) error { - connDestq := connDest.(*sqlite3.SQLiteConn) - connSrcq := connSrc.(*sqlite3.SQLiteConn) - bk, err := connDestq.Backup("main", connSrcq, "main") - if err != nil { - return fmt.Errorf("create backup db: %w", err) - } - for done, _ := bk.Step(-1); !done; { //nolint: revive - } - if err := bk.Finish(); err != nil { - return fmt.Errorf("finishing dump: %w", err) - } - return nil - }) - }) - if err != nil { - return fmt.Errorf("backing up: %w", err) - } - - return nil -} diff --git a/db/db_test.go b/db/db_test.go index 0fb2bf15..a5fef752 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -22,7 +22,7 @@ func TestGetSetting(t *testing.T) { key := SettingKey(randKey()) value := "howdy" - testDB, err := NewMock() + testDB, err := New("sqlite3://:memory:") if err != nil { t.Fatalf("error creating db: %v", err) } diff --git a/db/migrations.go b/db/migrations.go index 80304cf5..3fd28400 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -2,7 +2,6 @@ package db import ( - "context" "errors" "fmt" "log" @@ -20,7 +19,6 @@ import ( type MigrationContext struct { Production bool - DBPath string OriginalMusicPath string PlaylistsPath string PodcastsPath string @@ -59,7 +57,6 @@ func (db *DB) Migrate(ctx MigrationContext) error { construct(ctx, "202206101425", migrateUser), construct(ctx, "202207251148", migrateStarRating), construct(ctx, "202211111057", migratePlaylistsQueuesToFullID), - constructNoTx(ctx, "202212272312", backupDBPre016), construct(ctx, "202304221528", migratePlaylistsToM3U), construct(ctx, "202305301718", migratePlayCountToLength), construct(ctx, "202307281628", migrateAlbumArtistsMany2Many), @@ -179,12 +176,18 @@ func migrateAddGenre(tx *gorm.DB, _ MigrationContext) error { func migrateUpdateTranscodePrefIDX(tx *gorm.DB, _ MigrationContext) error { var hasIDX int - tx. - Select("1"). - Table("sqlite_master"). - Where("type = ?", "index"). - Where("name = ?", "idx_user_id_client"). - Count(&hasIDX) + if tx.Dialect().GetName() == "sqlite3" { + tx.Select("1"). + Table("sqlite_master"). + Where("type = ?", "index"). + Where("name = ?", "idx_user_id_client"). + Count(&hasIDX) + } else if tx.Dialect().GetName() == "postgres" { + tx.Select("1"). + Table("pg_indexes"). + Where("indexname = ?", "idx_user_id_client"). + Count(&hasIDX) + } if hasIDX == 1 { // index already exists return nil @@ -461,9 +464,15 @@ func migratePlaylistsQueuesToFullID(tx *gorm.DB, _ MigrationContext) error { if err := step.Error; err != nil { return fmt.Errorf("step migrate play_queues to full id: %w", err) } - step = tx.Exec(` + if tx.Dialect().GetName() == "postgres" { + step = tx.Exec(` + UPDATE play_queues SET newcurrent=('tr-' || current)::varchar[200]; + `) + } else { + step = tx.Exec(` UPDATE play_queues SET newcurrent=('tr-' || CAST(current AS varchar(10))); `) + } if err := step.Error; err != nil { return fmt.Errorf("step migrate play_queues to full id: %w", err) } @@ -590,7 +599,7 @@ func migrateAlbumArtistsMany2Many(tx *gorm.DB, _ MigrationContext) error { return fmt.Errorf("step insert from albums: %w", err) } - step = tx.Exec(`DROP INDEX idx_albums_tag_artist_id`) + step = tx.Exec(`DROP INDEX IF EXISTS idx_albums_tag_artist_id`) if err := step.Error; err != nil { return fmt.Errorf("step drop index: %w", err) } @@ -729,13 +738,6 @@ func migratePlaylistsPaths(tx *gorm.DB, ctx MigrationContext) error { return nil } -func backupDBPre016(tx *gorm.DB, ctx MigrationContext) error { - if !ctx.Production { - return nil - } - return Dump(context.Background(), tx, fmt.Sprintf("%s.%d.bak", ctx.DBPath, time.Now().Unix())) -} - func migrateAlbumTagArtistString(tx *gorm.DB, _ MigrationContext) error { return tx.AutoMigrate(Album{}).Error } @@ -770,12 +772,22 @@ func migrateArtistAppearances(tx *gorm.DB, _ MigrationContext) error { return fmt.Errorf("step transfer album artists: %w", err) } - step = tx.Exec(` + if tx.Dialect().GetName() == "sqlite3" { + step = tx.Exec(` INSERT OR IGNORE INTO artist_appearances (artist_id, album_id) SELECT track_artists.artist_id, tracks.album_id FROM track_artists JOIN tracks ON tracks.id=track_artists.track_id `) + } else { + step = tx.Exec(` + INSERT INTO artist_appearances (artist_id, album_id) + SELECT track_artists.artist_id, tracks.album_id + FROM track_artists + JOIN tracks ON tracks.id=track_artists.track_id + ON CONFLICT DO NOTHING + `) + } if err := step.Error; err != nil { return fmt.Errorf("step transfer album artists: %w", err) } @@ -795,7 +807,7 @@ func migrateTemporaryDisplayAlbumArtist(tx *gorm.DB, _ MigrationContext) error { return tx.Exec(` UPDATE albums SET tag_album_artist=( - SELECT group_concat(artists.name, ', ') + SELECT string_agg(artists.name, ', ') FROM artists JOIN album_artists ON album_artists.artist_id=artists.id AND album_artists.album_id=albums.id GROUP BY album_artists.album_id diff --git a/mockfs/mockfs.go b/mockfs/mockfs.go index 450c136b..b917b2a3 100644 --- a/mockfs/mockfs.go +++ b/mockfs/mockfs.go @@ -2,7 +2,6 @@ package mockfs import ( - "context" "errors" "fmt" "os" @@ -35,7 +34,7 @@ func NewWithExcludePattern(tb testing.TB, excludePattern string) *MockFS { func newMockFS(tb testing.TB, dirs []string, excludePattern string) *MockFS { tb.Helper() - dbc, err := db.NewMock() + dbc, err := db.New("sqlite3://:memory:") if err != nil { tb.Fatalf("create db: %v", err) } @@ -299,23 +298,6 @@ func (m *MockFS) SetTags(path string, cb func(*TagInfo)) { cb(m.tagReader.paths[absPath]) } -func (m *MockFS) DumpDB(suffix ...string) { - var p []string - p = append(p, - "gonic", "dump", - strings.ReplaceAll(m.t.Name(), string(filepath.Separator), "-"), - fmt.Sprint(time.Now().UnixNano()), - ) - p = append(p, suffix...) - - destPath := filepath.Join(os.TempDir(), strings.Join(p, "-")) - if err := db.Dump(context.Background(), m.db.DB, destPath); err != nil { - m.t.Fatalf("dumping db: %v", err) - } - - m.t.Error(destPath) -} - type tagReader struct { paths map[string]*TagInfo } diff --git a/scanner/scanner_fuzz_test.go b/scanner/scanner_fuzz_test.go index b4fc28ba..c48faedd 100644 --- a/scanner/scanner_fuzz_test.go +++ b/scanner/scanner_fuzz_test.go @@ -9,7 +9,6 @@ import ( "reflect" "testing" - _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/stretchr/testify/assert" "go.senan.xyz/gonic/mockfs" ) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index c49b6f86..a5bd38d0 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/jinzhu/gorm" - _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/server/ctrlsubsonic/handlers_by_folder.go b/server/ctrlsubsonic/handlers_by_folder.go index 2f20017b..600ea096 100644 --- a/server/ctrlsubsonic/handlers_by_folder.go +++ b/server/ctrlsubsonic/handlers_by_folder.go @@ -31,13 +31,13 @@ func (c *Controller) ServeGetIndexes(r *http.Request) *spec.Response { } var folders []*db.Album c.dbc. - Select("*, count(sub.id) child_count"). + Select("albums.*, count(sub.id) child_count"). Preload("AlbumStar", "user_id=?", user.ID). Preload("AlbumRating", "user_id=?", user.ID). Joins("LEFT JOIN albums sub ON albums.id=sub.parent_id"). Where("albums.parent_id IN ?", rootQ.SubQuery()). Group("albums.id"). - Order("albums.right_path COLLATE NOCASE"). + Order("albums.right_path"). Find(&folders) // [a-z#] -> 27 indexMap := make(map[string]*spec.Index, 27) @@ -80,7 +80,7 @@ func (c *Controller) ServeGetMusicDirectory(r *http.Request) *spec.Response { Where("parent_id=?", id.Value). Preload("AlbumStar", "user_id=?", user.ID). Preload("AlbumRating", "user_id=?", user.ID). - Order("albums.right_path COLLATE NOCASE"). + Order("albums.right_path"). Find(&childFolders) for _, ch := range childFolders { childrenObj = append(childrenObj, spec.NewTCAlbumByFolder(ch)) diff --git a/server/ctrlsubsonic/handlers_by_folder_test.go b/server/ctrlsubsonic/handlers_by_folder_test.go index 0efc4e24..6c0c4dd5 100644 --- a/server/ctrlsubsonic/handlers_by_folder_test.go +++ b/server/ctrlsubsonic/handlers_by_folder_test.go @@ -3,8 +3,6 @@ package ctrlsubsonic import ( "net/url" "testing" - - _ "github.com/jinzhu/gorm/dialects/sqlite" ) func TestGetIndexes(t *testing.T) { diff --git a/server/ctrlsubsonic/handlers_by_tags.go b/server/ctrlsubsonic/handlers_by_tags.go index 3f5da918..39e552bd 100644 --- a/server/ctrlsubsonic/handlers_by_tags.go +++ b/server/ctrlsubsonic/handlers_by_tags.go @@ -26,13 +26,13 @@ func (c *Controller) ServeGetArtists(r *http.Request) *spec.Response { user := r.Context().Value(CtxUser).(*db.User) var artists []*db.Artist q := c.dbc. - Select("*, count(album_artists.album_id) album_count"). + Select("artists.*, count(album_artists.album_id) album_count"). Joins("JOIN album_artists ON album_artists.artist_id=artists.id"). Preload("ArtistStar", "user_id=?", user.ID). Preload("ArtistRating", "user_id=?", user.ID). Preload("Info"). Group("artists.id"). - Order("artists.name COLLATE NOCASE") + Order("artists.name") if m := getMusicFolder(c.musicPaths, params); m != "" { q = q. Joins("JOIN albums ON albums.id=album_artists.album_id"). @@ -230,7 +230,7 @@ func (c *Controller) ServeSearchThree(r *http.Request) *spec.Response { // search artists var artists []*db.Artist q := c.dbc. - Select("*, count(albums.id) album_count"). + Select("artists.*, count(albums.id) album_count"). Group("artists.id") for _, s := range queries { q = q.Where(`name LIKE ? OR name_u_dec LIKE ?`, s, s) diff --git a/server/ctrlsubsonic/handlers_raw.go b/server/ctrlsubsonic/handlers_raw.go index d050ebe4..91578f57 100644 --- a/server/ctrlsubsonic/handlers_raw.go +++ b/server/ctrlsubsonic/handlers_raw.go @@ -256,7 +256,7 @@ func streamGetTransodePreference(dbc *db.DB, userID int, client string) (*db.Tra var pref db.TranscodePreference err := dbc. Where("user_id=?", userID). - Where("client COLLATE NOCASE IN (?)", []string{"*", client}). + Where("client IN (?)", []string{"*", client}). Order("client DESC"). // ensure "*" is last if it's there First(&pref). Error