feat: cache previously transformed request of dev server (#74)

* refactor: rename `transform` on cache service with `hash`

Because it misleads user defined transformation.

* refactor: split the logic that testing cache considering deps

* feat: cache previously transformed request of dev server

* test: add test case for dev server cache update

* test: add test case for additional data of cache

* fix: register nested deps' cache of target source

* chore: add comment

* chore: tweak comment

lib/cache-stream.js

@@ -1,92 +1,44 @@
 'use strict'
 
 const Transform = require('stream').Transform
+const DepCache = require('./dep-cache')
 
-module.exports = function (cache, depResolver, requestFile) {
+module.exports = function (cache, depResolver, readFile) {
   const contentsMap = new Map()
+  const depCache = new DepCache(cache, depResolver, getContent)
 
   function getContent (fileName) {
-    // If contentsMap has the previously loaded contents, just use it
-    if (contentsMap.has(fileName)) {
-      return contentsMap.get(fileName)
-    } else {
-      const contents = requestFile(fileName)
-      contentsMap.set(fileName, contents)
-      return contents
-    }
-  }
-
-  function shouldTransform (fileName, source) {
-    contentsMap.set(fileName, source)
-
-    // If original source is updated, it should be transformed
-    if (!cache.test(fileName, source)) return true
-
-    depResolver.register(fileName, source)
-    const newDeps = depResolver.getOutDeps(fileName)
-
-    // Loop through deps to compare its contents
-    let isUpdate = false
-    for (const fileName of newDeps) {
-      const contents = getContent(fileName)
-
-      // What should we do if possible deps are not found?
-      // For now, treat it as updated contents
-      // so that transformers can handle the error
-      if (contents == null) {
-        isUpdate = true
-      } else {
-        isUpdate |= !cache.test(fileName, contents)
-      }
-    }
-
-    return isUpdate
-  }
-
-  function walkNestedDeps (fileName, source) {
-    const footprints = new Set()
-
-    function loop (fileName, source) {
-      // Detect circular
-      if (footprints.has(fileName)) return
-      footprints.add(fileName)
-
-      // When `source` is empty value,
-      // the file is not exists so we must clear the cache.
-      if (source == null) {
-        cache.clear(fileName)
-        depResolver.clear(fileName)
-        return
-      }
-
-      cache.register(fileName, source)
-      depResolver.register(fileName, source)
-
-      depResolver.getOutDeps(fileName).forEach(fileName => {
-        loop(fileName, getContent(fileName))
-      })
+    // If contentsMap has the previously loaded content, just use it
+    const content = contentsMap.get(fileName)
+    if (content) {
+      return content
     }
 
-    return loop(fileName, source)
+    const loaded = readFile(fileName)
+    contentsMap.set(fileName, loaded)
+    return loaded
   }
 
   const stream = new Transform({
     objectMode: true,
     transform (file, encoding, callback) {
       const source = file.contents.toString()
 
-      if (shouldTransform(file.path, source)) {
+      contentsMap.set(file.path, source)
+      if (!depCache.test(file.path, source)) {
         this.push(file)
       }
 
       callback()
     }
   })
 
+  // Resolve all previously loaded files deps and register them to cache.
+  // We need lazily update the cache because it could block build targets
+  // if the nested deps are registered before building them.
   stream.on('finish', () => {
-    // Update cache and deps
     for (const entry of contentsMap.entries()) {
-      walkNestedDeps(entry[0], entry[1])
+      depCache.register(entry[0], entry[1])
     }
   })
 

lib/cache.js

@@ -4,32 +4,50 @@ const assert = require('assert')
 const util = require('./util')
 
 class Cache {
-  constructor (transform) {
+  constructor (hash) {
     this.map = {}
-    this.transform = transform || util.identity
+    this.hash = hash || util.identity
   }
 
-  register (filename, contents) {
+  get (filename) {
     assert(typeof filename === 'string', 'File name must be a string')
-    this.map[filename] = this.transform(contents)
+    const item = this.map[filename]
+    return item && item.data
+  }
+
+  register (filename, source, data) {
+    assert(typeof filename === 'string', 'File name must be a string')
+    let item = this.map[filename]
+    if (!item) {
+      item = this.map[filename] = {}
+    }
+
+    item.hash = this.hash(source)
+    item.data = data === undefined ? item.data : data
   }
 
   clear (filename) {
     assert(typeof filename === 'string', 'File name must be a string')
     delete this.map[filename]
   }
 
-  test (filename, contents) {
+  test (filename, source) {
     assert(typeof filename === 'string', 'File name must be a string')
-    return this.map[filename] === this.transform(contents)
+    const item = this.map[filename]
+    return !!item && item.hash === this.hash(source)
   }
 
+  /**
+   * We do not include cache data into serialized object
+   */
   serialize () {
-    return this.map
+    return util.mapValues(this.map, item => item.hash)
   }
 
   deserialize (map) {
-    this.map = map
+    this.map = util.mapValues(map, hash => {
+      return { hash }
+    })
   }
 }
 module.exports = Cache

lib/dep-cache.js

@@ -0,0 +1,90 @@
+'use strict'
+
+/**
+ * Similar with Cache but considering files dependencies.
+ */
+class DepCache {
+  constructor (cache, depResolver, readFile) {
+    this.cache = cache
+    this.depResolver = depResolver
+    this.readFile = readFile
+  }
+
+  get (filename) {
+    return this.cache.get(filename)
+  }
+
+  register (filename, source, data) {
+    const footprints = new Set()
+
+    const loop = (filename, source) => {
+      if (footprints.has(filename)) {
+        return
+      }
+      footprints.add(filename)
+
+      // #26
+      // When `source` is empty value,
+      // the file is not exists so we must clear the cache.
+      if (source == null) {
+        this.clear(filename)
+        return
+      }
+
+      this.cache.register(filename, source, data)
+      this.depResolver.register(filename, source)
+
+      this.depResolver.getOutDeps(filename).forEach(dep => {
+        const depSource = this.readFile(dep)
+        if (this.cache.test(dep, depSource)) {
+          return
+        }
+
+        loop(dep, depSource)
+      })
+    }
+
+    loop(filename, source, data)
+  }
+
+  clear (filename) {
+    // Only clear the target file cache because we cannot be sure
+    // the deps are really no longer useless for now.
+    // i.e. The user may refer them directly via `test` method.
+    this.cache.clear(filename)
+    this.depResolver.clear(filename)
+  }
+
+  /**
+   * Test the given file has the same contents.
+   * It also considers the dependencies. That means it is treated as
+   * updated when one of the dependenies is changed even if the target
+   * file itself is not.
+   */
+  test (fileName, source) {
+    // If original source is updated, it should not be the same.
+    if (!this.cache.test(fileName, source)) {
+      return false
+    }
+
+    const deps = this.depResolver.getOutDeps(fileName)
+
+    // Loop through deps to compare its contents
+    let isUpdate = false
+    for (const fileName of deps) {
+      const contents = this.readFile(fileName)
+
+      // What should we do if possible deps are not found?
+      // For now, treat it as updated contents
+      // so that transformers can handle the error
+      if (contents == null) {
+        isUpdate = true
+      } else {
+        isUpdate = isUpdate || !this.cache.test(fileName, contents)
+      }
+    }
+
+    return !isUpdate
+  }
+}
+module.exports = DepCache

lib/dev.js

@@ -3,10 +3,13 @@
 const url = require('url')
 const externalIp = require('./util').getIp()
 const loadConfig = require('./config').loadConfig
+const Cache = require('./cache')
 const DepResolver = require('./dep-resolver')
+const DepCache = require('./dep-cache')
 const create = require('./externals/browser-sync')
 const createWatcher = require('./externals/watcher')
 const DevLogger = require('./loggers/dev-logger')
+const util = require('./util')
 
 /**
  * The top level function for the dev server.
@@ -30,13 +33,16 @@ function dev (options, debug = {}) {
     console: debug.console
   })
 
+  const cache = new Cache()
+  // The state of DepResolver is shared between browser-sync and watcher
   const resolver = DepResolver.create(config)
+  const depCache = new DepCache(cache, resolver, util.readFileSync)
 
   const bs = create(config, {
     logLevel: 'silent',
     middleware: logMiddleware,
     open: !options._debug // Internal
-  }, resolver)
+  }, depCache)
 
   bs.emitter.on('init', () => {
     logger.startDevServer(config.port, externalIp)

lib/externals/browser-sync.js

@@ -4,6 +4,7 @@ const fs = require('fs')
 const path = require('path')
 const url = require('url')
 const vfs = require('vinyl-fs')
+const { Readable, Writable } = require('stream')
 const browserSync = require('browser-sync')
 const proxy = require('http-proxy-middleware')
 const mime = require('mime')
@@ -15,7 +16,7 @@ const defaultBsOptions = {
   notify: false
 }
 
-module.exports = (config, bsOptions, depResolver) => {
+module.exports = (config, bsOptions, depCache) => {
   const bs = browserSync.create()
 
   bsOptions = util.merge(defaultBsOptions, bsOptions)
@@ -25,7 +26,7 @@ module.exports = (config, bsOptions, depResolver) => {
   })
 
   bsOptions.server = config.output
-  injectMiddleware(bsOptions, transformer(config, depResolver))
+  injectMiddleware(bsOptions, transformer(config, depCache))
   config.proxy.forEach(p => {
     injectMiddleware(bsOptions, proxy(p.context, p.config))
   })
@@ -49,7 +50,7 @@ function injectMiddleware (options, middleware) {
   options.middleware = [middleware]
 }
 
-function transformer (config, depResolver) {
+function transformer (config, depCache) {
   const basePath = config.basePath
   return (req, res, next) => {
     const parsedPath = url.parse(req.url).pathname
@@ -82,16 +83,49 @@ function transformer (config, depResolver) {
       return redirect(res, reqPath + '/')
     }
 
-    const src = vfs.src(inputPath)
-      .on('data', file => {
-        depResolver.register(file.path, String(file.contents))
-      })
+    const responseFile = new Writable({
+      objectMode: true,
+
+      write (file, encoding, cb) {
+        const response = contents => {
+          res.setHeader('Content-Type', mime.getType(outputPath))
+          res.end(contents)
+        }
+
+        // If the inputPath hits the cache, just use cached output
+        const contentsStr = String(file.contents)
+        if (depCache.test(file.path, contentsStr)) {
+          const contents = depCache.get(file.path)
+          response(contents)
+          cb(null)
+          return
+        }
+
+        const inputProxy = new Readable({
+          objectMode: true,
+
+          read () {
+            this.push(file)
+            this.push(null)
+          }
+        })
+
+        rule.task(inputProxy)
+          .on('data', transformed => {
+            // Register the transformed contents into cache
+            depCache.register(
+              file.path,
+              contentsStr,
+              transformed.contents
+            )
+
+            response(transformed.contents)
+          })
+          .on('end', cb)
+      }
+    })
 
-    rule.task(src)
-      .on('data', file => {
-        res.setHeader('Content-Type', mime.getType(outputPath))
-        res.end(file.contents)
-      })
+    vfs.src(inputPath).pipe(responseFile)
   }
 }