feat: invalidation with wildcard

README.md

@@ -118,6 +118,47 @@ Options:
 Clear the cache. If `name` is specified, all the cache entries from the function defined with that name are cleared.
 If `arg` is specified, only the elements cached with the given `name` and `arg` are cleared.
 
+### `cache.invalidateAll(references, [storage])`
+
+`cache.invalidateAll` perform invalidation over the whole storage; if `storage` is not specified - using the same `name` as the defined function, invalidation is made over the default storage.
+
+`references` can be:
+
+* a single reference
+* an array of references (without wildcard)
+* a matching reference with wildcard, same logic for `memory` and `redis`
+
+Exmple
+
+```js
+const cache = createCache({ ttl: 60 })
+
+cache.define('fetchUser', {
+  references: (args, key, result) => result ? [`user:${result.id}`] : null
+}, (id) => database.find({ table: 'users', where: { id }}))
+
+cache.define('fetchCountries', {
+  storage: { type: 'memory', size: 256 },
+  references: (args, key, result) => [`countries`]
+}, (id) => database.find({ table: 'countries' }))
+
+// ...
+
+// invalidate all users from default storage
+cache.invalidateAll('user:*')
+
+// invalidate user 1 from default storage
+cache.invalidateAll('user:1')
+
+// invalidate user 1 and user 2 from default storage
+cache.invalidateAll(['user:1', 'user:2'])
+
+// note "fetchCountries" uses a different storage
+cache.invalidateAll('countries', 'fetchCountries')
+```
+
+See below how invalidation and references work.
+
 ## Invalidation
 
 Along with `time to live` invalidation of the cache entries, we can use invalidation by keys.  

src/cache.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { kValues, kStorage, kTTL, kOnDedupe, kOnError, kOnHit, kOnMiss } = require('./symbol')
+const { kValues, kStorage, kStorages, kTTL, kOnDedupe, kOnError, kOnHit, kOnMiss } = require('./symbol')
 const stringify = require('safe-stable-stringify')
 const createStorage = require('./storage')
 
@@ -40,7 +40,11 @@ class Cache {
     }
 
     this[kValues] = {}
+
     this[kStorage] = options.storage
+    this[kStorages] = new Map()
+    this[kStorages].set('_default', options.storage)
+
     this[kTTL] = options.ttl || 0
     this[kOnDedupe] = options.onDedupe || noop
     this[kOnError] = options.onError || noop
@@ -88,7 +92,14 @@ class Cache {
       throw new TypeError('references must be a function')
     }
 
-    const storage = opts.storage ? createStorage(opts.storage.type, opts.options) : this[kStorage]
+    let storage
+    if (opts.storage) {
+      storage = createStorage(opts.storage.type, opts.options)
+      this[kStorages].set(name, storage)
+    } else {
+      storage = this[kStorage]
+    }
+
     const ttl = opts.ttl || this[kTTL]
     const onDedupe = opts.onDedupe || this[kOnDedupe]
     const onError = opts.onError || this[kOnError]
@@ -143,10 +154,16 @@ class Cache {
       throw new Error(`${name} is not defined in the cache`)
     }
 
-    // TODO validate references?
-
     return this[kValues][name].invalidate(references)
   }
+
+  async invalidateAll(references, storage = '_default') {
+    if (!this[kStorages].has(storage)) {
+      throw new Error(`${storage} storage is not defined in the cache`)
+    }
+    const s = this[kStorages].get(storage)
+    await s.invalidate(references)
+  }
 }
 
 class Wrapper {

src/storage/memory.js

@@ -3,7 +3,7 @@
 const LRUCache = require('mnemonist/lru-cache')
 const nullLogger = require('abstract-logging')
 const StorageInterface = require('./interface')
-const { findMatchingIndexes, findNotMatching, bsearchIndex } = require('../util')
+const { findMatchingIndexes, findNotMatching, bsearchIndex, wildcardMatch } = require('../util')
 
 const DEFAULT_CACHE_SIZE = 1024
 
@@ -241,7 +241,7 @@ class StorageMemory extends StorageInterface {
   }
 
   /**
-   * @param {string[]} references
+   * @param {string|string[]} references
    * @returns {string[]} removed keys
    */
   invalidate (references) {
@@ -252,31 +252,80 @@ class StorageMemory extends StorageInterface {
 
     this.log.debug({ msg: 'acd/storage/memory.invalidate', references })
 
+    if (Array.isArray(references)) {
+      return this._invalidateReferences(references)
+    }
+    return this._invalidateReference(references)
+  }
+
+  /**
+   * @param {string[]} references
+   * @returns {string[]} removed keys
+   */
+  _invalidateReferences(references) {
     const removed = []
     for (let i = 0; i < references.length; i++) {
       const reference = references[i]
       const keys = this.referencesKeys.get(reference)
-      this.log.debug({ msg: 'acd/storage/memory.invalidate, remove keys on reference', reference, keys })
+      this.log.debug({ msg: 'acd/storage/memory._invalidateReferences, remove keys on reference', reference, keys })
       if (!keys) {
         continue
       }
 
       for (let j = 0; j < keys.length; j++) {
         const key = keys[j]
-        this.log.debug({ msg: 'acd/storage/memory.invalidate, remove key on reference', reference, key })
+        this.log.debug({ msg: 'acd/storage/memory._invalidateReferences, remove key on reference', reference, key })
         // istanbul ignore next
         if (this._removeKey(key)) {
           removed.push(key)
         }
       }
 
-      this.log.debug({ msg: 'acd/storage/memory.invalidate, remove references of', reference, keys })
+      this.log.debug({ msg: 'acd/storage/memory._invalidateReferences, remove references of', reference, keys })
       this._removeReferences([...keys])
     }
 
     return removed
   }
 
+  /**
+   * @param {string} reference
+   * @returns {string[]} removed keys
+   */
+  _invalidateReference(reference) {
+    if (reference.includes('*')) {
+      const references = []
+      for (const key of this.referencesKeys.keys()) {
+        if (wildcardMatch(reference, key)) {
+          references.push(key)
+        }
+      }
+      return this._invalidateReferences(references)
+    }
+
+    const keys = this.referencesKeys.get(reference)
+    const removed = []
+    this.log.debug({ msg: 'acd/storage/memory._invalidateReference, remove keys on reference', reference, keys })
+
+    if (!keys) {
+      return removed
+    }
+
+    for (let j = 0; j < keys.length; j++) {
+      const key = keys[j]
+      this.log.debug({ msg: 'acd/storage/memory._invalidateReference, remove key on reference', reference, key })
+      // istanbul ignore next
+      if (this._removeKey(key)) {
+        removed.push(key)
+      }
+    }
+
+    this.log.debug({ msg: 'acd/storage/memory._invalidateReference, remove references of', reference, keys })
+    this._removeReferences([...keys])
+
+    return removed
+  }
+
   /**
    * remove all entries if name is not provided
    * remove entries where key starts with name if provided

src/storage/redis.js

@@ -80,7 +80,7 @@ class StorageRedis extends StorageInterface {
    * @param {?string[]} references
    */
   async set (key, value, ttl, references) {
-    // TODO can keys contains * or other special chars?
+    // TODO validate keys, can't contain * or other special chars
     this.log.debug({ msg: 'acd/storage/redis.set key', key, value, ttl, references })
 
     ttl = Number(ttl)
@@ -160,7 +160,7 @@ class StorageRedis extends StorageInterface {
   }
 
   /**
-   * @param {string[]} references
+   * @param {string|string[]} references
    * @returns {string[]} removed keys
    */
   async invalidate (references) {
@@ -172,33 +172,74 @@ class StorageRedis extends StorageInterface {
     this.log.debug({ msg: 'acd/storage/redis.invalidate', references })
 
     try {
-      const reads = references.map(reference => ['smembers', this.getReferenceKeyLabel(reference)])
-      const keys = await this.store.pipeline(reads).exec()
-
-      this.log.debug({ msg: 'acd/storage/redis.invalidate keys', keys })
-
-      const writes = []
-      const removed = []
-      for (let i = 0; i < keys.length; i++) {
-        const key0 = keys[i][1]
-        this.log.debug({ msg: 'acd/storage/redis.invalidate got keys to be invalidated', keys: key0 })
-        for (let j = 0; j < key0.length; j++) {
-          const key1 = key0[j]
-          this.log.debug({ msg: 'acd/storage/redis.del key' + key1 })
-          removed.push(key1)
-          writes.push(['del', key1])
-        }
+      if (Array.isArray(references)) {
+        return await this._invalidateReferences(references)
       }
-
-      await this.store.pipeline(writes).exec()
-      await this.clearReferences(removed)
-      return removed
+      return await this._invalidateReference(references)
     } catch (err) {
       this.log.error({ msg: 'acd/storage/redis.invalidate error', err, references })
       return []
     }
   }
 
+  /**
+   * @param {string[]} references
+   * @param {[bool=true]} mapReferences
+   * @returns {string[]} removed keys
+   */
+  async _invalidateReferences(references, mapReferences = true) {
+    const reads = references.map(reference => ['smembers', mapReferences ? this.getReferenceKeyLabel(reference) : reference])
+    const keys = await this.store.pipeline(reads).exec()
+
+    this.log.debug({ msg: 'acd/storage/redis._invalidateReferences keys', keys })
+
+    const writes = []
+    const removed = []
+    for (let i = 0; i < keys.length; i++) {
+      const key0 = keys[i][1]
+      this.log.debug({ msg: 'acd/storage/redis._invalidateReferences got keys to be invalidated', keys: key0 })
+      for (let j = 0; j < key0.length; j++) {
+        const key1 = key0[j]
+        this.log.debug({ msg: 'acd/storage/redis._invalidateReferences del key' + key1 })
+        removed.push(key1)
+        writes.push(['del', key1])
+      }
+    }
+
+    await this.store.pipeline(writes).exec()
+    await this.clearReferences(removed)
+    return removed
+  }
+
+  /**
+   * @param {string} reference
+   * @returns {string[]} removed keys
+   */
+  async _invalidateReference(reference) {
+    let keys
+    if (reference.includes('*')) {
+      const references = await this.store.keys(this.getReferenceKeyLabel(reference))
+      return this._invalidateReferences(references, false)
+    } else {
+      keys = await this.store.smembers(this.getReferenceKeyLabel(reference))
+    }
+
+    this.log.debug({ msg: 'acd/storage/redis._invalidateReference keys', keys })
+
+    const writes = []
+    const removed = []
+    for (let i = 0; i < keys.length; i++) {
+      const key0 = keys[i]
+      this.log.debug({ msg: 'acd/storage/redis._invalidateReference del key' + key0 })
+      removed.push(key0)
+      writes.push(['del', key0])
+    }
+
+    await this.store.pipeline(writes).exec()
+    await this.clearReferences(removed)
+    return removed
+  }
+
   /**
    * @param {string} name
    */

src/symbol.js

@@ -2,10 +2,11 @@
 
 const kValues = Symbol('values')
 const kStorage = Symbol('kStorage')
+const kStorages = Symbol('kStorages')
 const kTTL = Symbol('kTTL')
 const kOnDedupe = Symbol('kOnDedupe')
 const kOnError = Symbol('kOnError')
 const kOnHit = Symbol('kOnHit')
 const kOnMiss = Symbol('kOnMiss')
 
-module.exports = { kValues, kStorage, kTTL, kOnDedupe, kOnError, kOnHit, kOnMiss }
+module.exports = { kValues, kStorage, kStorages, kTTL, kOnDedupe, kOnError, kOnHit, kOnMiss }

src/util.js

@@ -94,10 +94,44 @@ function randomSubset (array, size) {
 
   return result
 }
+
+/**
+ * @param {!string} value substring to search in content, supporting wildcard 
+ * @param {!string} content string to search in
+ * @return {boolean} true if value is in content
+ * @example wildcardMatch("1*5", "12345") > true
+ * @example wildcardMatch("1*6", "12345") > false
+ */
+function wildcardMatch(value, content) {
+  if (value === '*') return true
+  if (value.length === content.length && value === content) return true
+
+  let i = 0, j = 0
+  while (i < value.length && j < content.length) {
+    if (value[i] === content[j]) {
+      i++
+      j++
+      continue
+    }
+    if (value[i] === '*') {
+      if (value[i + 1] === content[j]) {
+        i++
+        continue
+      }
+      j++
+      continue
+    }
+    return false
+  }
+
+  return i >= value.length - 1
+}
+
 module.exports = {
   findNotMatching,
   findMatchingIndexes,
   bsearchIndex,
+  wildcardMatch,
 
   randomSubset
 }