Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add simple cache warmer, disabled by default
- Loading branch information
Showing
10 changed files
with
267 additions
and
28 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
package core | ||
|
||
import ( | ||
"context" | ||
"io/ioutil" | ||
|
||
"github.com/deluan/navidrome/conf" | ||
"github.com/deluan/navidrome/core/pool" | ||
"github.com/deluan/navidrome/log" | ||
) | ||
|
||
type CacheWarmer interface { | ||
AddAlbum(ctx context.Context, albumID string) | ||
Flush(ctx context.Context) | ||
} | ||
|
||
func NewCacheWarmer(cache ArtworkCache, artwork Artwork) CacheWarmer { | ||
w := &warmer{ | ||
artwork: artwork, | ||
cache: cache, | ||
albums: map[string]struct{}{}, | ||
} | ||
p, err := pool.NewPool("artwork", 3, &artworkItem{}, w.execute) | ||
if err != nil { | ||
log.Error(context.Background(), "Error creating pool for Album Artwork Cache Warmer", err) | ||
} else { | ||
w.pool = p | ||
} | ||
|
||
return w | ||
} | ||
|
||
type warmer struct { | ||
pool *pool.Pool | ||
artwork Artwork | ||
cache ArtworkCache | ||
albums map[string]struct{} | ||
} | ||
|
||
func (w *warmer) AddAlbum(ctx context.Context, albumID string) { | ||
if albumID == "" { | ||
return | ||
} | ||
w.albums[albumID] = struct{}{} | ||
} | ||
|
||
func (w *warmer) Flush(ctx context.Context) { | ||
if conf.Server.DevPreCacheAlbumArtwork { | ||
if w.pool == nil || len(w.albums) == 0 { | ||
return | ||
} | ||
log.Info(ctx, "Pre-caching album artworks", "numAlbums", len(w.albums)) | ||
for id := range w.albums { | ||
w.pool.Submit(artworkItem{albumID: id}) | ||
} | ||
} | ||
w.albums = map[string]struct{}{} | ||
} | ||
|
||
func (w *warmer) execute(workload interface{}) { | ||
ctx := context.Background() | ||
item := workload.(artworkItem) | ||
log.Trace(ctx, "Pre-caching album artwork", "albumID", item.albumID) | ||
err := w.artwork.Get(ctx, item.albumID, 0, ioutil.Discard) | ||
if err != nil { | ||
log.Warn("Error pre-caching artwork from album", "id", item.albumID, err) | ||
} | ||
} | ||
|
||
type artworkItem struct { | ||
albumID string | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,99 @@ | ||
package pool | ||
|
||
type Executor func(workload interface{}) | ||
|
||
type Pool struct { | ||
name string | ||
item interface{} | ||
workers []worker | ||
exec Executor | ||
//queue *dque.DQue | ||
queue chan work // receives jobs to send to workers | ||
end chan bool // when receives bool stops workers | ||
} | ||
|
||
func NewPool(name string, workerCount int, item interface{}, exec Executor) (*Pool, error) { | ||
p := &Pool{ | ||
name: name, | ||
item: item, | ||
exec: exec, | ||
queue: make(chan work), | ||
end: make(chan bool), | ||
} | ||
|
||
//q, err := dque.NewOrOpen(name, filepath.Join(conf.Server.DataFolder, "queues", name), 50, p.itemBuilder) | ||
//if err != nil { | ||
// return nil, err | ||
//} | ||
//p.queue = q | ||
for i := 0; i < workerCount; i++ { | ||
worker := worker{ | ||
p: p, | ||
id: i, | ||
channel: make(chan work), | ||
workerChannel: workerChannel, | ||
end: make(chan bool)} | ||
worker.Start() | ||
p.workers = append(p.workers, worker) | ||
} | ||
|
||
// start pool | ||
go func() { | ||
for { | ||
select { | ||
case <-p.end: | ||
for _, w := range p.workers { | ||
w.Stop() // stop worker | ||
} | ||
return | ||
case work := <-p.queue: | ||
worker := <-workerChannel // wait for available channel | ||
worker <- work // dispatch work to worker | ||
} | ||
} | ||
}() | ||
return p, nil | ||
} | ||
|
||
func (p *Pool) Submit(workload interface{}) { | ||
p.queue <- work{workload} | ||
} | ||
|
||
//func (p *Pool) itemBuilder() interface{} { | ||
// t := reflect.TypeOf(p.item) | ||
// return reflect.New(t).Interface() | ||
//} | ||
// | ||
var workerChannel = make(chan chan work) | ||
|
||
type work struct { | ||
workload interface{} | ||
} | ||
|
||
type worker struct { | ||
id int | ||
p *Pool | ||
workerChannel chan chan work // used to communicate between dispatcher and workers | ||
channel chan work | ||
end chan bool | ||
} | ||
|
||
// start worker | ||
func (w *worker) Start() { | ||
go func() { | ||
for { | ||
w.workerChannel <- w.channel // when the worker is available place channel in queue | ||
select { | ||
case job := <-w.channel: // worker has received job | ||
w.p.exec(job.workload) // do work | ||
case <-w.end: | ||
return | ||
} | ||
} | ||
}() | ||
} | ||
|
||
// end worker | ||
func (w *worker) Stop() { | ||
w.end <- true | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package pool | ||
|
||
import ( | ||
"testing" | ||
|
||
"github.com/deluan/navidrome/log" | ||
"github.com/deluan/navidrome/tests" | ||
. "github.com/onsi/ginkgo" | ||
. "github.com/onsi/gomega" | ||
) | ||
|
||
func TestCore(t *testing.T) { | ||
tests.Init(t, false) | ||
log.SetLevel(log.LevelCritical) | ||
RegisterFailHandler(Fail) | ||
RunSpecs(t, "Core Suite") | ||
} | ||
|
||
var _ = Describe("Pool", func() { | ||
|
||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
package migration | ||
|
||
import ( | ||
"database/sql" | ||
"os" | ||
"path/filepath" | ||
|
||
"github.com/deluan/navidrome/conf" | ||
"github.com/pressly/goose" | ||
) | ||
|
||
func init() { | ||
goose.AddMigration(Up20201025222059, Down20201025222059) | ||
} | ||
|
||
func Up20201025222059(tx *sql.Tx) error { | ||
cacheFolder := filepath.Join(conf.Server.DataFolder, "cache") | ||
notice(tx, "Purging all cache entries, as the format of the cache changed.") | ||
return os.RemoveAll(cacheFolder) | ||
} | ||
|
||
func Down20201025222059(tx *sql.Tx) error { | ||
// This code is executed when the migration is rolled back. | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters