feature/mongo cache (#180)

* add siteID to cache Get

* indirect option setters

* add mongo cache with tests, add Key and Flusher

* lint: minor warns

* workaround for cache parallel test

* repeater in mongo cache

* missing repeater vendor

* fix nop cache

* add cache mongo benchmark

* wired mongo cache, single opts group for mongo

* disable goconst

* stop cache repeated on not found error

* use local mongo for tests in travis

.travis.yml

@@ -3,6 +3,9 @@ install:
   - docker-compose --version
 
 script:
+  - docker run -d --name=mongo mongo:3.6 && sleep 3
+  - export MONGO_TEST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
+  - echo "running mongo on $MONGO_TEST"
   - docker build
       --build-arg COVERALLS_TOKEN=$COVERALLS_TOKEN
       --build-arg CI=$CI
@@ -16,5 +19,7 @@ script:
       --build-arg TRAVIS_PULL_REQUEST_SHA=$TRAVIS_PULL_REQUEST_SHA
       --build-arg TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG
       --build-arg TRAVIS_TAG=$TRAVIS_TAG
-      --build-arg MONGO_TEST=$MONGO_REMARK_TEST
+      --build-arg MONGO_TEST=$MONGO_TEST
       .
+  - docker rm -f mongo
+

Dockerfile

@@ -37,7 +37,7 @@ RUN echo "mongo=${MONGO_TEST}" >> /etc/hosts
 RUN if [ -z "$SKIP_BACKEND_TEST" ] ; then \
     if [ -f .mongo ] ; then export MONGO_TEST=$(cat .mongo) ; fi && \
     gometalinter --disable-all --deadline=300s --vendor --enable=vet --enable=vetshadow --enable=golint \
-    --enable=staticcheck --enable=ineffassign --enable=goconst --enable=errcheck --enable=unconvert \
+    --enable=staticcheck --enable=ineffassign --enable=errcheck --enable=unconvert \
     --enable=deadcode  --enable=gosimple --enable=gas --exclude=test --exclude=mock --exclude=vendor ./... ; \
     else echo "skip backend linters" ; fi
 

backend/app/main.go

@@ -28,14 +28,14 @@ import (
 )
 
 // Opts with command line flags and env
-// nolint:maligned
 type Opts struct {
 	SecretKey string `long:"secret" env:"SECRET" required:"true" description:"secret key"`
 	RemarkURL string `long:"url" env:"REMARK_URL" required:"true" description:"url to remark"`
 
 	Store  StoreGroup  `group:"store" namespace:"store" env-namespace:"STORE"`
 	Avatar AvatarGroup `group:"avatar" namespace:"avatar" env-namespace:"AVATAR"`
 	Cache  CacheGroup  `group:"cache" namespace:"cache" env-namespace:"CACHE"`
+	Mongo  MongoGroup  `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 
 	Sites          []string      `long:"site" env:"SITE" default:"remark" description:"site names" env-delim:","`
 	Admins         []string      `long:"admin" env:"ADMIN" description:"admin(s) names" env-delim:","`
@@ -79,7 +79,6 @@ type StoreGroup struct {
 		Path    string        `long:"path" env:"PATH" default:"./var" description:"parent dir for bolt files"`
 		Timeout time.Duration `long:"timeout" env:"TIMEOUT" default:"30s" description:"bolt timeout"`
 	} `group:"bolt" namespace:"bolt" env-namespace:"BOLT"`
-	Mongo MongoOpts `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
 }
 
 // AvatarGroup defines options group for avatar params
@@ -88,22 +87,21 @@ type AvatarGroup struct {
 	FS   struct {
 		Path string `long:"path" env:"PATH" default:"./var/avatars" description:"avatars location"`
 	} `group:"fs" namespace:"fs" env-namespace:"FS"`
-	Mongo  MongoOpts `group:"mongo" namespace:"mongo" env-namespace:"MONGO"`
-	RszLmt int       `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
+	RszLmt int `long:"rsz-lmt" env:"RESIZE" default:"0" description:"max image size for resizing avatars on save"`
 }
 
 // CacheGroup defines options group for cache params
 type CacheGroup struct {
-	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"redis" default:"mem"`
+	Type string `long:"type" env:"TYPE" description:"type of cache" choice:"mem" choice:"mongo" default:"mem"`
 	Max  struct {
 		Items int   `long:"items" env:"ITEMS" default:"1000" description:"max cached items"`
 		Value int   `long:"value" env:"VALUE" default:"65536" description:"max size of cached value"`
 		Size  int64 `long:"size" env:"SIZE" default:"50000000" description:"max size of total cache"`
 	} `group:"max" namespace:"max" env-namespace:"MAX"`
 }
 
-// MongoOpts holds all mongo params
-type MongoOpts struct {
+// MongoGroup holds all mongo params, used by store, avatar and cache
+type MongoGroup struct {
 	URL string `long:"url" env:"URL" description:"mongo url"`
 	DB  string `long:"db" env:"DB" default:"remark42" description:"mongo database"`
 }
@@ -163,7 +161,7 @@ func New(opts Opts) (*Application, error) {
 		return nil, errors.Errorf("invalid remark42 url %s", opts.RemarkURL)
 	}
 
-	storeEngine, err := makeDataStore(opts.Store, opts.Sites)
+	storeEngine, err := makeDataStore(opts.Store, opts.Mongo, opts.Sites)
 	if err != nil {
 		return nil, err
 	}
@@ -176,8 +174,7 @@ func New(opts Opts) (*Application, error) {
 		Admins:         opts.Admins,
 	}
 
-	loadingCache, err := cache.NewMemoryCache(cache.MaxCacheSize(opts.Cache.Max.Size), cache.MaxValSize(opts.Cache.Max.Value),
-		cache.MaxKeys(opts.Cache.Max.Items))
+	loadingCache, err := makeCache(opts.Cache, opts.Mongo)
 	if err != nil {
 		return nil, err
 	}
@@ -186,7 +183,7 @@ func New(opts Opts) (*Application, error) {
 	jwtService := auth.NewJWT(opts.SecretKey, strings.HasPrefix(opts.RemarkURL, "https://"),
 		opts.Auth.TTL.JWT, opts.Auth.TTL.Cookie)
 
-	avatarStore, err := makeAvatarStore(opts.Avatar)
+	avatarStore, err := makeAvatarStore(opts.Avatar, opts.Mongo)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make avatar store")
 	}
@@ -296,7 +293,7 @@ func (a *Application) activateBackup(ctx context.Context) {
 }
 
 // makeDataStore creates store for all sites
-func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interface, err error) {
+func makeDataStore(group StoreGroup, mg MongoGroup, siteNames []string) (result engine.Interface, err error) {
 	switch group.Type {
 	case "bolt":
 		if err = makeDirs(group.Bolt.Path); err != nil {
@@ -308,36 +305,53 @@ func makeDataStore(group StoreGroup, siteNames []string) (result engine.Interfac
 		}
 		result, err = engine.NewBoltDB(bolt.Options{Timeout: group.Bolt.Timeout}, sites...)
 	case "mongo":
-		mgServer, e := makeMongo(group.Mongo)
+		mgServer, e := makeMongo(mg)
 		if e != nil {
 			return result, errors.Wrap(e, "failed to create mongo server")
 		}
-		conn := mongo.NewConnection(mgServer, group.Mongo.DB, "")
+		conn := mongo.NewConnection(mgServer, mg.DB, "")
 		result, err = engine.NewMongo(conn, 500, 100*time.Millisecond)
 	default:
 		return nil, errors.Errorf("unsupported store type %s", group.Type)
 	}
 	return result, errors.Wrap(err, "can't initialize data store")
 }
 
-func makeAvatarStore(group AvatarGroup) (avatar.Store, error) {
+func makeAvatarStore(group AvatarGroup, mg MongoGroup) (avatar.Store, error) {
 	switch group.Type {
 	case "fs":
 		if err := makeDirs(group.FS.Path); err != nil {
 			return nil, err
 		}
 		return avatar.NewLocalFS(group.FS.Path, group.RszLmt), nil
 	case "mongo":
-		mgServer, err := makeMongo(group.Mongo)
+		mgServer, err := makeMongo(mg)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create mongo server")
 		}
-		conn := mongo.NewConnection(mgServer, group.Mongo.DB, "")
+		conn := mongo.NewConnection(mgServer, mg.DB, "")
 		return avatar.NewGridFS(conn, group.RszLmt), nil
 	}
 	return nil, errors.Errorf("unsupported avatar store type %s", group.Type)
 }
 
+func makeCache(group CacheGroup, mg MongoGroup) (cache.LoadingCache, error) {
+	switch group.Type {
+	case "mem":
+		return cache.NewMemoryCache(cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
+			cache.MaxKeys(group.Max.Items))
+	case "mongo":
+		mgServer, err := makeMongo(mg)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create mongo server")
+		}
+		conn := mongo.NewConnection(mgServer, mg.DB, "cache")
+		return cache.NewMongoCache(conn, cache.MaxCacheSize(group.Max.Size), cache.MaxValSize(group.Max.Value),
+			cache.MaxKeys(group.Max.Items))
+	}
+	return nil, errors.Errorf("unsupported cache type %s", group.Type)
+}
+
 // mkdir -p for all dirs
 func makeDirs(dirs ...string) error {
 
@@ -367,11 +381,11 @@ func makeDirs(dirs ...string) error {
 	return nil
 }
 
-func makeMongo(mopts MongoOpts) (result *mongo.Server, err error) {
-	if mopts.URL == "" {
+func makeMongo(mg MongoGroup) (result *mongo.Server, err error) {
+	if mg.URL == "" {
 		return nil, errors.New("no mongo URL provided")
 	}
-	return mongo.NewServerWithURL(mopts.URL, 10*time.Second)
+	return mongo.NewServerWithURL(mg.URL, 10*time.Second)
 }
 
 func makeAuthProviders(jwtService *auth.JWT, avatarProxy *proxy.Avatar, ds *service.DataStore, opts Opts) []auth.Provider {

backend/app/rest/api/admin.go

@@ -57,7 +57,7 @@ func (a *admin) deleteCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete comment")
 		return
 	}
-	a.cache.Flush(locator.SiteID, locator.URL)
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last"))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, JSON{"id": id, "locator": locator})
 }
@@ -73,7 +73,7 @@ func (a *admin) deleteUserCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't delete user")
 		return
 	}
-	a.cache.Flush(siteID, userID)
+	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID})
 }
@@ -118,7 +118,7 @@ func (a *admin) deleteMeRequestCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't delete user")
 		return
 	}
-	a.cache.Flush(claims.SiteID, claims.User.ID)
+	a.cache.Flush(cache.Flusher(claims.SiteID).Scopes(claims.SiteID, claims.User.ID, "last"))
 	render.Status(r, http.StatusOK)
 	render.JSON(w, r, JSON{"user_id": claims.User.ID, "site_id": claims.SiteID})
 }
@@ -140,7 +140,7 @@ func (a *admin) setBlockCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set blocking status")
 		return
 	}
-	a.cache.Flush(siteID, userID)
+	a.cache.Flush(cache.Flusher(siteID).Scopes(userID, siteID))
 	render.JSON(w, r, JSON{"user_id": userID, "site_id": siteID, "block": blockStatus})
 }
 
@@ -177,7 +177,7 @@ func (a *admin) setReadOnlyCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set readonly status")
 		return
 	}
-	a.cache.Flush(locator.SiteID)
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, locator.SiteID))
 	render.JSON(w, r, JSON{"locator": locator, "read-only": roStatus})
 }
 
@@ -191,7 +191,7 @@ func (a *admin) setVerifyCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set verify status")
 		return
 	}
-	a.cache.Flush(siteID, userID)
+	a.cache.Flush(cache.Flusher(siteID).Scopes(siteID, userID))
 	render.JSON(w, r, JSON{"user": userID, "verified": verifyStatus})
 }
 
@@ -206,7 +206,7 @@ func (a *admin) setPinCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't set pin status")
 		return
 	}
-	a.cache.Flush(locator.URL)
+	a.cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL))
 	render.JSON(w, r, JSON{"id": commentID, "locator": locator, "pin": pinStatus})
 }
 

backend/app/rest/api/migrator.go

@@ -98,7 +98,7 @@ func (m *Migrator) importCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "import failed")
 		return
 	}
-	m.Cache.Flush(siteID)
+	m.Cache.Flush(cache.Flusher(siteID).Scopes(siteID))
 
 	render.Status(r, http.StatusCreated)
 	render.JSON(w, r, JSON{"status": "ok", "size": size})

backend/app/rest/api/migrator_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/umputun/remark/backend/app/migrator"
+	"github.com/umputun/remark/backend/app/rest/cache"
 	"github.com/umputun/remark/backend/app/store/engine"
 	"github.com/umputun/remark/backend/app/store/service"
 )
@@ -111,7 +112,7 @@ func prepImportSrv(t *testing.T) (svc *Migrator, ts *httptest.Server) {
 		DisqusImporter: &migrator.Disqus{DataStore: dataStore},
 		NativeImporter: &migrator.Remark{DataStore: dataStore},
 		NativeExported: &migrator.Remark{DataStore: dataStore},
-		Cache:          &mockCache{},
+		Cache:          &cache.Nop{},
 		SecretKey:      "123456",
 	}
 

backend/app/rest/api/rest_private.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/umputun/remark/backend/app/rest"
 	"github.com/umputun/remark/backend/app/rest/auth"
+	"github.com/umputun/remark/backend/app/rest/cache"
 	"github.com/umputun/remark/backend/app/store"
 	"github.com/umputun/remark/backend/app/store/service"
 )
@@ -70,7 +71,8 @@ func (s *Rest) createCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusInternalServerError, err, "can't load created comment")
 		return
 	}
-	s.Cache.Flush(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID)
+	s.Cache.Flush(cache.Flusher(comment.Locator.SiteID).
+		Scopes(comment.Locator.URL, "last", comment.User.ID, comment.Locator.SiteID))
 
 	render.Status(r, http.StatusCreated)
 	render.JSON(w, r, &finalComment)
@@ -121,7 +123,7 @@ func (s *Rest) updateCommentCtrl(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	s.Cache.Flush(locator.URL, "last", user.ID)
+	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, "last", user.ID))
 	render.JSON(w, r, res)
 }
 
@@ -155,7 +157,7 @@ func (s *Rest) voteCtrl(w http.ResponseWriter, r *http.Request) {
 		rest.SendErrorJSON(w, r, http.StatusBadRequest, err, "can't vote for comment")
 		return
 	}
-	s.Cache.Flush(locator.URL, comment.User.ID)
+	s.Cache.Flush(cache.Flusher(locator.SiteID).Scopes(locator.URL, comment.User.ID))
 	render.JSON(w, r, JSON{"id": comment.ID, "score": comment.Score})
 }