feat(storers): Add a new NutsMemcached storage backend built on NutsDB

In this commit NutsMemcached behave just like Nuts.

configurationtypes/types.go

@@ -224,6 +224,7 @@ type DefaultCache struct {
 	Etcd                CacheProvider `json:"etcd" yaml:"etcd"`
 	Mode                string        `json:"mode" yaml:"mode"`
 	Nuts                CacheProvider `json:"nuts" yaml:"nuts"`
+	NutsMemcached       CacheProvider `json:"nuts_memcached" yaml:"nuts_memcached"`
 	Olric               CacheProvider `json:"olric" yaml:"olric"`
 	Redis               CacheProvider `json:"redis" yaml:"redis"`
 	Port                Port          `json:"port" yaml:"port"`
@@ -286,6 +287,11 @@ func (d *DefaultCache) GetNuts() CacheProvider {
 	return d.Nuts
 }
 
+// GetNutsMemcached returns nuts_memcached configuration
+func (d *DefaultCache) GetNutsMemcached() CacheProvider {
+	return d.NutsMemcached
+}
+
 // GetOlric returns olric configuration
 func (d *DefaultCache) GetOlric() CacheProvider {
 	return d.Olric
@@ -341,6 +347,7 @@ type DefaultCacheInterface interface {
 	GetEtcd() CacheProvider
 	GetMode() string
 	GetNuts() CacheProvider
+	GetNutsMemcached() CacheProvider
 	GetOlric() CacheProvider
 	GetRedis() CacheProvider
 	GetHeaders() []string

pkg/storage/nutsMemcachedProvider.go

@@ -0,0 +1,273 @@
+package storage
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+
+	t "github.com/darkweak/souin/configurationtypes"
+	"github.com/darkweak/souin/pkg/rfc"
+	"github.com/darkweak/souin/pkg/storage/types"
+	"github.com/imdario/mergo"
+	"github.com/nutsdb/nutsdb"
+	"go.uber.org/zap"
+)
+
+var nutsMemcachedInstanceMap = map[string]*nutsdb.DB{}
+
+// NutsMemcached provider type
+type NutsMemcached struct {
+	*nutsdb.DB
+	stale  time.Duration
+	logger *zap.Logger
+}
+
+// const (
+// 	bucket    = "souin-bucket"
+// 	nutsLimit = 1 << 16
+// )
+
+// func sanitizeProperties(m map[string]interface{}) map[string]interface{} {
+// 	iotas := []string{"RWMode", "StartFileLoadingMode"}
+// 	for _, i := range iotas {
+// 		if v := m[i]; v != nil {
+// 			currentMode := nutsdb.FileIO
+// 			switch v {
+// 			case 1:
+// 				currentMode = nutsdb.MMap
+// 			}
+// 			m[i] = currentMode
+// 		}
+// 	}
+
+// 	for _, i := range []string{"SegmentSize", "NodeNum", "MaxFdNumsInCache"} {
+// 		if v := m[i]; v != nil {
+// 			m[i], _ = v.(int64)
+// 		}
+// 	}
+
+// 	if v := m["EntryIdxMode"]; v != nil {
+// 		m["EntryIdxMode"] = nutsdb.HintKeyValAndRAMIdxMode
+// 		switch v {
+// 		case 1:
+// 			m["EntryIdxMode"] = nutsdb.HintKeyAndRAMIdxMode
+// 		}
+// 	}
+
+// 	if v := m["SyncEnable"]; v != nil {
+// 		m["SyncEnable"] = true
+// 		if b, ok := v.(bool); ok {
+// 			m["SyncEnable"] = b
+// 		} else if s, ok := v.(string); ok {
+// 			m["SyncEnable"], _ = strconv.ParseBool(s)
+// 		}
+// 	}
+
+// 	return m
+// }
+
+// NutsConnectionFactory function create new Nuts instance
+func NutsMemcachedConnectionFactory(c t.AbstractConfigurationInterface) (types.Storer, error) {
+	dc := c.GetDefaultCache()
+	nutsConfiguration := dc.GetNutsMemcached()
+	nutsOptions := nutsdb.DefaultOptions
+	nutsOptions.Dir = "/tmp/souin-nuts-memcached"
+	if nutsConfiguration.Configuration != nil {
+		var parsedNuts nutsdb.Options
+		nutsConfiguration.Configuration = sanitizeProperties(nutsConfiguration.Configuration.(map[string]interface{}))
+		if b, e := json.Marshal(nutsConfiguration.Configuration); e == nil {
+			if e = json.Unmarshal(b, &parsedNuts); e != nil {
+				c.GetLogger().Sugar().Error("Impossible to parse the configuration for the Nuts provider", e)
+			}
+		}
+
+		if err := mergo.Merge(&nutsOptions, parsedNuts, mergo.WithOverride); err != nil {
+			c.GetLogger().Sugar().Error("An error occurred during the nutsOptions merge from the default options with your configuration.")
+		}
+	} else {
+		nutsOptions.RWMode = nutsdb.MMap
+		if nutsConfiguration.Path != "" {
+			nutsOptions.Dir = nutsConfiguration.Path
+		}
+	}
+
+	if instance, ok := nutsMemcachedInstanceMap[nutsOptions.Dir]; ok && instance != nil {
+		return &NutsMemcached{
+			DB:     instance,
+			stale:  dc.GetStale(),
+			logger: c.GetLogger(),
+		}, nil
+	}
+
+	db, e := nutsdb.Open(nutsOptions)
+
+	if e != nil {
+		c.GetLogger().Sugar().Error("Impossible to open the Nuts DB.", e)
+		return nil, e
+	}
+
+	instance := &NutsMemcached{
+		DB:     db,
+		stale:  dc.GetStale(),
+		logger: c.GetLogger(),
+	}
+	nutsMemcachedInstanceMap[nutsOptions.Dir] = instance.DB
+
+	return instance, nil
+}
+
+// Name returns the storer name
+func (provider *NutsMemcached) Name() string {
+	return "NUTS_MEMCACHED"
+}
+
+// ListKeys method returns the list of existing keys
+func (provider *NutsMemcached) ListKeys() []string {
+	keys := []string{}
+
+	e := provider.DB.View(func(tx *nutsdb.Tx) error {
+		e, _ := tx.GetAll(bucket)
+		for _, k := range e {
+			if !strings.Contains(string(k.Key), surrogatePrefix) {
+				keys = append(keys, string(k.Key))
+			}
+		}
+		return nil
+	})
+
+	if e != nil {
+		return []string{}
+	}
+
+	return keys
+}
+
+// MapKeys method returns the map of existing keys
+func (provider *NutsMemcached) MapKeys(prefix string) map[string]string {
+	keys := map[string]string{}
+
+	e := provider.DB.View(func(tx *nutsdb.Tx) error {
+		e, _ := tx.GetAll(bucket)
+		for _, k := range e {
+			if strings.HasPrefix(string(k.Key), prefix) {
+				nk, _ := strings.CutPrefix(string(k.Key), prefix)
+				keys[nk] = string(k.Value)
+			}
+		}
+		return nil
+	})
+
+	if e != nil {
+		return map[string]string{}
+	}
+
+	return keys
+}
+
+// Get method returns the populated response if exists, empty response then
+func (provider *NutsMemcached) Get(key string) (item []byte) {
+	_ = provider.DB.View(func(tx *nutsdb.Tx) error {
+		i, e := tx.Get(bucket, []byte(key))
+		if i != nil {
+			item = i.Value
+		}
+		return e
+	})
+
+	return
+}
+
+// Prefix method returns the populated response if exists, empty response then
+func (provider *NutsMemcached) Prefix(key string, req *http.Request, validator *rfc.Revalidator) *http.Response {
+	var result *http.Response
+
+	_ = provider.DB.View(func(tx *nutsdb.Tx) error {
+		prefix := []byte(key)
+
+		if entries, err := tx.PrefixSearchScan(bucket, prefix, "^({|$)", 0, 50); err != nil {
+			return err
+		} else {
+			for _, entry := range entries {
+				if varyVoter(key, req, string(entry.Key)) {
+					if res, err := http.ReadResponse(bufio.NewReader(bytes.NewBuffer(entry.Value)), req); err == nil {
+						rfc.ValidateETag(res, validator)
+						if validator.Matched {
+							provider.logger.Sugar().Debugf("The stored key %s matched the current iteration key ETag %+v", string(entry.Key), validator)
+							result = res
+							return nil
+						}
+
+						provider.logger.Sugar().Debugf("The stored key %s didn't match the current iteration key ETag %+v", string(entry.Key), validator)
+					} else {
+						provider.logger.Sugar().Errorf("An error occured while reading response for the key %s: %v", string(entry.Key), err)
+					}
+				}
+			}
+		}
+		return nil
+	})
+
+	return result
+}
+
+// Set method will store the response in Nuts provider
+func (provider *NutsMemcached) Set(key string, value []byte, url t.URL, duration time.Duration) error {
+	if duration == 0 {
+		duration = url.TTL.Duration
+	}
+
+	err := provider.DB.Update(func(tx *nutsdb.Tx) error {
+		return tx.Put(bucket, []byte(key), value, uint32(duration.Seconds()))
+	})
+
+	if err != nil {
+		provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err)
+		return err
+	}
+
+	err = provider.DB.Update(func(tx *nutsdb.Tx) error {
+		return tx.Put(bucket, []byte(StalePrefix+key), value, uint32((provider.stale + duration).Seconds()))
+	})
+
+	if err != nil {
+		provider.logger.Sugar().Errorf("Impossible to set value into Nuts, %v", err)
+	}
+
+	return nil
+}
+
+// Delete method will delete the response in Nuts provider if exists corresponding to key param
+func (provider *NutsMemcached) Delete(key string) {
+	_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
+		return tx.Delete(bucket, []byte(key))
+	})
+}
+
+// DeleteMany method will delete the responses in Nuts provider if exists corresponding to the regex key param
+func (provider *NutsMemcached) DeleteMany(key string) {
+	_ = provider.DB.Update(func(tx *nutsdb.Tx) error {
+		if entries, err := tx.PrefixSearchScan(bucket, []byte(""), key, 0, nutsLimit); err != nil {
+			return err
+		} else {
+			for _, entry := range entries {
+				_ = tx.Delete(bucket, entry.Key)
+			}
+		}
+		return nil
+	})
+}
+
+// Init method will
+func (provider *NutsMemcached) Init() error {
+	return nil
+}
+
+// Reset method will reset or close provider
+func (provider *NutsMemcached) Reset() error {
+	return provider.DB.Update(func(tx *nutsdb.Tx) error {
+		return tx.DeleteBucket(1, bucket)
+	})
+}

pkg/storage/storage.go

@@ -26,6 +26,7 @@ var storageMap = map[string]StorerInstanciator{
 	"olric":          OlricConnectionFactory,
 	"embedded_olric": EmbeddedOlricConnectionFactory,
 	"nuts":           NutsConnectionFactory,
+	"nuts_memcached": NutsMemcachedConnectionFactory,
 	"badger":         BadgerConnectionFactory,
 }
 
@@ -44,6 +45,8 @@ func getStorageNameFromConfiguration(configuration configurationtypes.AbstractCo
 		}
 	} else if configuration.GetDefaultCache().GetNuts().Configuration != nil || configuration.GetDefaultCache().GetNuts().Path != "" {
 		return "nuts"
+	} else if configuration.GetDefaultCache().GetNutsMemcached().Configuration != nil || configuration.GetDefaultCache().GetNutsMemcached().Path != "" {
+		return "nuts_memcached"
 	}
 
 	return "badger"

plugins/caddy/configuration.go

@@ -40,6 +40,8 @@ type DefaultCache struct {
 	Etcd configurationtypes.CacheProvider `json:"etcd"`
 	// NutsDB provider configuration.
 	Nuts configurationtypes.CacheProvider `json:"nuts"`
+	// NutsMemcached provider configuration.
+	NutsMemcached configurationtypes.CacheProvider `json:"nuts_memcached"`
 	// Regex to exclude cache.
 	Regex configurationtypes.Regex `json:"regex"`
 	// Storage providers chaining and order.
@@ -102,6 +104,11 @@ func (d *DefaultCache) GetNuts() configurationtypes.CacheProvider {
 	return d.Nuts
 }
 
+// GetNutsMemcached returns nuts_memcached configuration
+func (d *DefaultCache) GetNutsMemcached() configurationtypes.CacheProvider {
+	return d.NutsMemcached
+}
+
 // GetOlric returns olric configuration
 func (d *DefaultCache) GetOlric() configurationtypes.CacheProvider {
 	return d.Olric
@@ -491,6 +498,24 @@ func parseConfiguration(cfg *Configuration, h *caddyfile.Dispenser, isGlobal boo
 					}
 				}
 				cfg.DefaultCache.Nuts = provider
+			case "nuts_memcached":
+				provider := configurationtypes.CacheProvider{}
+				for nesting := h.Nesting(); h.NextBlock(nesting); {
+					directive := h.Val()
+					switch directive {
+					case "url":
+						urlArgs := h.RemainingArgs()
+						provider.URL = urlArgs[0]
+					case "path":
+						urlArgs := h.RemainingArgs()
+						provider.Path = urlArgs[0]
+					case "configuration":
+						provider.Configuration = parseCaddyfileRecursively(h)
+					default:
+						return h.Errf("unsupported nuts directive: %s", directive)
+					}
+				}
+				cfg.DefaultCache.NutsMemcached = provider
 			case "olric":
 				cfg.DefaultCache.Distributed = true
 				provider := configurationtypes.CacheProvider{}