fix(studio-be): fix cache invalidation being discarded by flow service (

#229)

* fix(studio-be): fix cache invalidation being discarded by flow service

* better cache invalidation handling

* remove console

packages/studio-be/src/core/config/config-loader.ts

@@ -33,8 +33,9 @@ export class ConfigProvider {
     @inject(TYPES.Logger) private logger: Logger,
     @inject(TYPES.ObjectCache) private cache: ObjectCache
   ) {
-    this.cache.events.on('invalidation', async key => {
-      if (key === 'object::data/global/botpress.config.json' || key === 'file::data/global/botpress.config.json') {
+    this.cache.events.on('invalidation', async (key: string) => {
+      const re = /(object|file)::(?:data\/)?global\/botpress\.config\.json/g
+      if (key.match(re)) {
         this._botpressConfigCache = undefined
         const config = await this.getBotpressConfig()
 

packages/studio-be/src/core/dialog/flow/flow-service.ts

@@ -93,12 +93,12 @@ export class FlowService {
   private _listenForCacheInvalidation() {
     this.cache.events.on('invalidation', async key => {
       try {
-        const matches = key.match(/^([A-Z0-9-_]+)::data\/bots\/([A-Z0-9-_]+)\/flows\/([\s\S]+(flow)\.json)/i)
+        const matches = key.match(/^([A-Z0-9-_]+)::(?:data\/)?bots\/([A-Z0-9-_]+)\/flows\/([\s\S]+(flow)\.json)/i)
 
         if (matches && matches.length >= 2) {
-          const [key, type, botId, flowName] = matches
+          const [_key, type, botId, flowName] = matches
           if (type === 'file' || type === 'object') {
-            await this.forBot(botId).handleInvalidatedCache(flowName, type === 'file')
+            await this.forBot(botId).handleInvalidatedCache(flowName)
           }
         }
       } catch (err) {
@@ -128,7 +128,6 @@ export class FlowService {
 
 export class ScopedFlowService {
   private cache: ArrayCache<string, FlowView>
-  private expectedSavesCache: LRUCache<string, number>
 
   constructor(
     private botId: string,
@@ -142,9 +141,8 @@ export class ScopedFlowService {
   ) {
     this.cache = new ArrayCache<string, FlowView>(
       x => x.name,
-      (x, prevKey, newKey) => ({ ...x, name: newKey, location: newKey })
+      (x, _prevKey, newKey) => ({ ...x, name: newKey, location: newKey })
     )
-    this.expectedSavesCache = new LRUCache({ max: 100, maxAge: ms('20s') })
   }
 
   public async localInvalidateFlow(key: string, flow?: FlowView, newKey?: string) {
@@ -169,26 +167,20 @@ export class ScopedFlowService {
     }
   }
 
-  public async handleInvalidatedCache(flowName: string, isFromFile: boolean) {
+  public async handleInvalidatedCache(flowName: string) {
     const flowPath = this.toFlowPath(flowName)
-    const expectedSaves = this.expectedSavesCache.get(flowPath)
 
-    if (!expectedSaves) {
-      if (await this.ghost.fileExists(FLOW_DIR, flowPath)) {
-        const flow = await this.parseFlow(flowPath)
-        await this.localInvalidateFlow(flowPath, flow)
-      } else {
-        await this.localInvalidateFlow(flowPath, undefined)
-      }
-    } else {
-      if (!isFromFile) {
-        this.expectedSavesCache.set(flowPath, expectedSaves - 1)
-      }
+    // fix an issue when creating a bot where the .flow.json is written but not the .ui.json because of the locking mechanism
+    if (!(await this.ghost.fileExists(FLOW_DIR, this.toUiPath(flowPath)))) {
+      return
     }
-  }
 
-  private setExpectedSaves(flowName: string, amount: number) {
-    this.expectedSavesCache.set(flowName, amount)
+    if (await this.ghost.fileExists(FLOW_DIR, flowPath)) {
+      const flow = await this.parseFlow(flowPath)
+      await this.localInvalidateFlow(flowPath, flow)
+    } else {
+      await this.localInvalidateFlow(flowPath, undefined)
+    }
   }
 
   async loadAll(): Promise<FlowView[]> {
@@ -334,14 +326,8 @@ export class ScopedFlowService {
 
     const isNew = !flowFiles.find(x => flow.location === x)
 
-    // TODO: remove this when we improve the cache
-    // the onFlowChange function called inside prepareSaveFlow can cause
-    // cache events to be fired. We aren't interested in catching any of those
-    this.setExpectedSaves(flow.name, 999999)
     const { flowPath, uiPath, flowContent, uiContent } = await this.prepareSaveFlow(flow, isNew)
 
-    // TODO: remove this when we improve the cache
-    this.setExpectedSaves(flow.name, 2)
     this.invalidateFlow(flow.name, flow)
 
     await Promise.all([
@@ -359,9 +345,7 @@ export class ScopedFlowService {
 
     const uiPath = this.toUiPath(fileToDelete)
 
-    // TODO: remove this when we improve the cache
     this.invalidateFlow(flowName)
-    this.setExpectedSaves(flowName, 2)
 
     await Promise.all([this.ghost.deleteFile(FLOW_DIR, fileToDelete!), this.ghost.deleteFile(FLOW_DIR, uiPath)])
 
@@ -384,9 +368,7 @@ export class ScopedFlowService {
       throw new Error(`New flow name ${newName} is already in use`)
     }
 
-    // TODO: remove this when we improve the cache
     this.invalidateFlow(previousName, undefined, newName)
-    this.setExpectedSaves(newName, 2)
 
     const previousUiName = this.toUiPath(fileToRename)
     const newUiName = this.toUiPath(newName)

packages/studio-be/src/core/user-code/hints/hints-service.ts

@@ -29,7 +29,9 @@ export interface Hint {
   parentObject?: string
 }
 
-const invalidationFilePrefix = 'string::data/'
+// Matches strings::data/* and strings::*
+// And capture what is after
+const invalidationFilePrefix = /^string::(?:data\/)?(.+)$/
 
 @injectable()
 export class HintsService {
@@ -50,15 +52,16 @@ export class HintsService {
   }
 
   private _listenForCacheInvalidation() {
-    this.cache.events.on('invalidation', async key => {
-      if (!key.startsWith(invalidationFilePrefix)) {
+    this.cache.events.on('invalidation', async (key: string) => {
+      const matches = key.match(invalidationFilePrefix)
+      if (!matches || matches.length < 2) {
         return
       }
 
       // We let invalidation happens first
       // This is necessary because the ghost relies on the cache when reading file content
       await Promise.delay(100)
-      const filePath = key.substr(invalidationFilePrefix.length)
+      const filePath = matches[1]
       this.hints[filePath] = await this.indexFile(filePath)
     })
   }