Issue #2270 - Cache implementation (#2293)

docs/user-guide/cli.md

@@ -40,6 +40,12 @@ Using `bar/mySpecialConfig.json` as config, with quiet mode on, to lint all `.cs
 stylelint "foo/**/*.css bar/*.css" -q -f json --config bar/mySpecialConfig.json > myJsonReport.json
 ```
 
+Caching processed `.scss` files in order to operate only on changed ones in the `foo` directory, using the `cache` and `cache-location` options:
+
+```
+stylelint "foo/**/*.scss" --cache --cache-location "/Users/user/.stylelintcache/"
+```
+
 Linting all the `.scss` files in the `foo` directory, using the `syntax` option:
 
 ```shell

docs/user-guide/node-api.md

@@ -77,6 +77,22 @@ If `true`, all disable comments (e.g. `/* stylelint-disable block-no-empty */`)
 
 You can use this option to see what your linting results would be like without those exceptions.
 
+## `cache`
+
+Store the info about processed files in order to only operate on the changed ones the next time you run stylelint. Enabling this option can dramatically improve stylelint's speed, because only changed files will be linted.
+
+By default, the cache is stored in `.stylelintcache` in `process.cwd()`. To change this, use the `cacheLocation` option.
+
+**Note:** If you run stylelint with `cache` and then run stylelint without `cache`, the `.stylelintcache` file will be deleted. This is necessary because we have to assume that `.stylelintcache` was invalidated by that second command.
+
+## `cacheLocation`
+
+A path to a file or directory to be used for `cache`. Only meaningful alongside `cache`. If no location is specified, `.stylelintcache` will be created in `process.cwd()`.
+
+If a directory is specified, a cache file will be created inside the specified folder. The name of the file will be based on the hash of `process.cwd()` (e.g. `.cache_hashOfCWD`). This allows stylelint to reuse a single location for a variety of caches from different projects.
+
+**Note:** If the directory of `cacheLocation` does not exist, make sure you add a trailing `/` on \*nix systems or `\` on Windows. Otherwise, the path will be assumed to be a file.
+
 ### `reportNeedlessDisables`
 
 If `true`, `ignoreDisables` will also be set to `true` and the returned data will contain a `needlessDisables` property, whose value is an array of objects, one for each source, with tells you which stylelint-disable comments are not blocking a lint warning.

lib/__tests__/fixtures/cache/valid.css

@@ -0,0 +1 @@
+/* This file will not cause a linting error */

lib/__tests__/standalone-cache.test.js

@@ -0,0 +1,162 @@
+"use strict"
+
+const path = require("path")
+const standalone = require("../standalone")
+const hash = require("../utils/hash")
+const fixturesPath = path.join(__dirname, "fixtures")
+const fsExtra = require("fs-promise")
+const fileExists = require("file-exists-promise")
+
+const cwd = process.cwd()
+const invalidFile = path.join(fixturesPath, "empty-block.css")
+const validFile = path.join(fixturesPath, "cache", "valid.css")
+const newFileDest = path.join(fixturesPath, "cache", "newFile.css")
+
+// Config object is getting mutated internally.
+// Return new object of the same structure to
+// make sure config doesn't change between runs.
+function getConfig() {
+  return {
+    files: path.join(fixturesPath, "cache", "*.css"),
+    config: {
+      rules: { "block-no-empty": true, "color-no-invalid-hex": true },
+    },
+    cache: true,
+  }
+}
+
+describe("standalone cache", () => {
+  const expectedCacheFilePath = path.join(cwd, ".stylelintcache")
+
+  beforeEach(() => {
+    // Initial run to warm up the cache
+    return standalone(getConfig())
+  })
+
+  afterEach(() => {
+    // Clean up after each test case
+    return Promise.all([
+      fsExtra.remove(expectedCacheFilePath),
+      fsExtra.remove(newFileDest),
+    ])
+  })
+
+  it("cache file is created at $CWD/.stylelintcache", () => {
+    // Ensure cache file exists
+    return fileExists(expectedCacheFilePath).then(isFileExist => {
+      expect(!!isFileExist).toBe(true)
+      return fsExtra.readJson(expectedCacheFilePath)
+    }).then(cacheFile => {
+      // Ensure cache file contains only linted css file
+      expect(typeof cacheFile[validFile] === "object").toBe(true)
+      expect(typeof cacheFile[newFileDest] === "undefined").toBe(true)
+    })
+  })
+
+  it("only changed files are linted", () => {
+    // Add "changed" file
+    return fsExtra.copy(validFile, newFileDest).then(() => {
+      // Next run should lint only changed files
+      return standalone(getConfig())
+    }).then(output => {
+      // Ensure only changed files are linted
+      const isValidFileLinted = !!output.results.find(file => file.source === validFile)
+      const isNewFileLinted = !!output.results.find(file => file.source === newFileDest)
+      expect(isValidFileLinted).toBe(false)
+      expect(isNewFileLinted).toBe(true)
+      // Ensure cache file contains linted css files
+      return fsExtra.readJson(expectedCacheFilePath)
+    }).then(cachedFiles => {
+      expect(typeof cachedFiles[validFile] === "object").toBe(true)
+      expect(typeof cachedFiles[newFileDest] === "object").toBe(true)
+    })
+  })
+
+  it("all files are linted on config change", () => {
+    const changedConfig = getConfig()
+    changedConfig.config.rules["block-no-empty"] = false
+    return fsExtra.copy(validFile, newFileDest).then(() => {
+      // All file should be re-linted as config has changed
+      return standalone(changedConfig)
+    }).then(output => {
+      // Ensure all files are re-linted
+      const isValidFileLinted = !!output.results.find(file => file.source === validFile)
+      const isNewFileLinted = !!output.results.find(file => file.source === newFileDest)
+      expect(isValidFileLinted).toBe(true)
+      expect(isNewFileLinted).toBe(true)
+    })
+  })
+
+  it("invalid files are not cached", () => {
+    return fsExtra.copy(invalidFile, newFileDest).then(() => {
+      // Should lint only changed files
+      return standalone(getConfig())
+    }).then((output) => {
+      expect(output.errored).toBe(true)
+      // Ensure only changed files are linted
+      const isValidFileLinted = !!output.results.find(file => file.source === validFile)
+      const isInvalidFileLinted = !!output.results.find(file => file.source === newFileDest)
+      expect(isValidFileLinted).toBe(false)
+      expect(isInvalidFileLinted).toBe(true)
+      // Ensure cache file doesn't contain invalid file
+      return fsExtra.readJsonSync(expectedCacheFilePath)
+    }).then(cachedFiles => {
+      expect(typeof cachedFiles[validFile] === "object").toBe(true)
+      expect(typeof cachedFiles[newFileDest] === "undefined").toBe(true)
+    })
+  })
+  it("cache file is removed when cache is disabled", () => {
+    const noCacheConfig = getConfig()
+
+    noCacheConfig.cache = false
+    let cacheFileExists = true
+    return standalone(noCacheConfig).then(() => {
+      return fileExists(expectedCacheFilePath).then(() => {
+        throw new Error(`Cache file is supposed to be removed, ${expectedCacheFilePath} is found instead`)
+      }).catch(() => {
+        cacheFileExists = false
+        expect(cacheFileExists).toBe(false)
+      })
+    })
+  })
+})
+describe("standalone cache uses cacheLocation", () => {
+  const cacheLocationFile = path.join(fixturesPath, "cache", ".cachefile")
+  const cacheLocationDir = path.join(fixturesPath, "cache")
+  const expectedCacheFilePath = path.join(cacheLocationDir, `.stylelintcache_${hash(cwd)}`)
+  afterEach(() => {
+    // clean up after each test
+    return Promise.all([
+      fsExtra.remove(expectedCacheFilePath),
+      fsExtra.remove(cacheLocationFile),
+    ])
+  })
+  it("cacheLocation is a file", () => {
+    const config = getConfig()
+    config.cacheLocation = cacheLocationFile
+    return standalone(config).then(() => {
+      // Ensure cache file is created
+      return fileExists(cacheLocationFile)
+    }).then(fileStats => {
+      expect(!!fileStats).toBe(true)
+      // Ensure cache file contains cached entity
+      return fsExtra.readJson(cacheLocationFile)
+    }).then(cacheFile => {
+      expect(typeof cacheFile[validFile] === "object").toBe(true)
+    })
+  })
+  it("cacheLocation is a directory", () => {
+    const config = getConfig()
+    config.cacheLocation = cacheLocationDir
+    return standalone(config).then(() => {
+      return fileExists(expectedCacheFilePath)
+    }).then(cacheFileStats => {
+      // Ensure cache file is created
+      expect(!!cacheFileStats).toBe(true)
+      // Ensure cache file contains cached entity
+      return fsExtra.readJson(expectedCacheFilePath)
+    }).then(cacheFile => {
+      expect(typeof cacheFile[validFile] === "object").toBe(true)
+    })
+  })
+})

lib/cli.js

@@ -92,6 +92,23 @@ const meowOptions = {
       --ignore-disables, --id
 
         Ignore styleline-disable comments.
+        
+      --cache                       [default: false]
+
+        Store the info about processed files in order to only operate on the 
+        changed ones the next time you run stylelint. By default, the cache 
+        is stored in "./.stylelintcache". To adjust this, use --cache-location.
+
+      --cache-location              [default: '.stylelintcache']
+
+        Path to a file or directory to be used for the cache location.
+        Default is "./.stylelintcache". If a directory is specified, a cache 
+        file will be created inside the specified folder, with a name derived
+        from a hash of the current working directory.
+
+        If the directory for the cache does not exist, make sure you add a trailing "/" 
+        on \*nix systems or "\" on Windows. Otherwise the path will be assumed to be a file.
+         
 
       --formatter, -f               [default: "string"]
 
@@ -183,6 +200,14 @@ if (cli.flags.allowEmptyInput) {
   optionsBase.allowEmptyInput = cli.flags.allowEmptyInput
 }
 
+if (cli.flags.cache) {
+  optionsBase.cache = true
+}
+
+if (cli.flags.cacheLocation) {
+  optionsBase.cacheLocation = cli.flags.cacheLocation
+}
+
 const reportNeedlessDisables = cli.flags.reportNeedlessDisables
 
 if (reportNeedlessDisables) {

lib/standalone.js

@@ -6,6 +6,10 @@ const createStylelint = require("./createStylelint")
 const globby = require("globby")
 const needlessDisables = require("./needlessDisables")
 const alwaysIgnoredGlobs = require("./alwaysIgnoredGlobs")
+const FileCache = require("./utils/FileCache")
+const debug = require("debug")("stylelint:standalone")
+const pkg = require("../package.json")
+const hash = require("./utils/hash")
 
 module.exports = function (options/*: Object */)/*: Promise<stylelint$standaloneReturnValue>*/ {
   const files = options.files
@@ -22,6 +26,11 @@ module.exports = function (options/*: Object */)/*: Promise<stylelint$standalone
   const syntax = options.syntax
   const customSyntax = options.customSyntax
   const allowEmptyInput = options.allowEmptyInput
+  const cacheLocation = options.cacheLocation
+  const useCache = options.cache || false
+  let fileCache
+
+  const startTime = Date.now()
 
   const isValidCode = typeof code === "string"
   if (!files && !isValidCode || files && (code || isValidCode)) {
@@ -74,6 +83,17 @@ module.exports = function (options/*: Object */)/*: Promise<stylelint$standalone
     alwaysIgnoredGlobs.map(file => "!" + file)
   )
 
+  if (useCache) {
+    const stylelintVersion = pkg.version
+    const hashOfConfig = hash(`${stylelintVersion}_${JSON.stringify(config)}`)
+    fileCache = new FileCache(cacheLocation, hashOfConfig)
+  } else {
+    // No need to calculate hash here, we just want to delete cache file.
+    fileCache = new FileCache(cacheLocation)
+    // Remove cache file if cache option is disabled
+    fileCache.destroy()
+  }
+
   return globby(fileList).then(filePaths => {
     if (!filePaths.length) {
       if (allowEmptyInput === undefined || !allowEmptyInput) {
@@ -97,14 +117,27 @@ module.exports = function (options/*: Object */)/*: Promise<stylelint$standalone
       }
     }
 
-    const getStylelintResults = filePaths.map(filePath => {
+    let absoluteFilePaths = filePaths.map(filePath => {
       const absoluteFilepath = (!path.isAbsolute(filePath))
         ? path.join(process.cwd(), filePath)
-        : filePath
+        : path.normalize(filePath)
+      return absoluteFilepath
+    })
+
+    if (useCache) {
+      absoluteFilePaths = absoluteFilePaths.filter(fileCache.hasFileChanged.bind(fileCache))
+    }
+
+    const getStylelintResults = absoluteFilePaths.map(absoluteFilepath => {
+      debug(`Processing ${absoluteFilepath}`)
       return stylelint._lintSource({
         filePath: absoluteFilepath,
       }).then(postcssResult => {
-        return stylelint._createStylelintResult(postcssResult, filePath)
+        if (postcssResult.stylelint.stylelintError && useCache) {
+          debug(`${absoluteFilepath} contains linting errors and will not be cached.`)
+          fileCache.removeEntry(absoluteFilepath)
+        }
+        return stylelint._createStylelintResult(postcssResult, absoluteFilepath)
       }).catch(handleError)
     })
 
@@ -121,6 +154,10 @@ module.exports = function (options/*: Object */)/*: Promise<stylelint$standalone
     if (reportNeedlessDisables) {
       returnValue.needlessDisables = needlessDisables(stylelintResults)
     }
+    if (useCache) {
+      fileCache.reconcile()
+    }
+    debug(`Linting complete in ${Date.now() - startTime}ms`)
     return returnValue
   }
 }

lib/utils/FileCache.js

@@ -0,0 +1,48 @@
+/* @flow */
+"use strict"
+
+const fileEntryCache = require("file-entry-cache")
+const path = require("path")
+const debug = require("debug")("stylelint:file-cache")
+const getCacheFile = require("./getCacheFile")
+
+const DEFAULT_CACHE_LOCATION = "./.stylelintcache"
+const DEFAULT_HASH = ""
+
+function FileCache(cacheLocation/*: String */, hashOfConfig/*: ?String */) {
+  const cacheFile = path.resolve(getCacheFile(cacheLocation || DEFAULT_CACHE_LOCATION, process.cwd()))
+  debug(`Cache file is created at ${cacheFile}`)
+  this._fileCache = fileEntryCache.create(cacheFile)
+  this._hashOfConfig = hashOfConfig || DEFAULT_HASH
+}
+
+FileCache.prototype.hasFileChanged = function (absoluteFilepath) {
+  // Get file descriptor compares current metadata against cached
+  // one and stores the result to "changed" prop.
+  const descriptor = this._fileCache.getFileDescriptor(absoluteFilepath)
+  const meta = descriptor.meta || {}
+  const changed = descriptor.changed || meta.hashOfConfig !== this._hashOfConfig
+  if (!changed) {
+    debug(`Skip linting ${absoluteFilepath}. File hasn't changed.`)
+  }
+  // Mutate file descriptor object and store config hash to each file.
+  // Running lint with different config should invalidate the cache.
+  if (meta.hashOfConfig !== this._hashOfConfig) {
+    meta.hashOfConfig = this._hashOfConfig
+  }
+  return changed
+}
+
+FileCache.prototype.reconcile = function () {
+  this._fileCache.reconcile()
+}
+
+FileCache.prototype.destroy = function () {
+  this._fileCache.destroy()
+}
+
+FileCache.prototype.removeEntry = function (absoluteFilepath) {
+  this._fileCache.removeEntry(absoluteFilepath)
+}
+
+module.exports = FileCache