load-paths-handler.js

/* global emit */

const async = require('async')
const fs = require('fs')
const os = require('os')
const path = require('path')
const {GitRepository} = require('atom')
const {Minimatch} = require('minimatch')
const childProcess = require('child_process')
const { rgPath } = require('vscode-ripgrep')

const PathsChunkSize = 100

// Use the unpacked path if the ripgrep binary is in asar archive.
const realRgPath = rgPath.replace(/\bapp\.asar\b/, 'app.asar.unpacked')

// Define the maximum number of concurrent crawling processes based on the number of CPUs
// with a maximum value of 8 and minimum of 1.
const MaxConcurrentCrawls = Math.min(Math.max(os.cpus().length - 1, 8), 1)

const emittedPaths = new Set()

class PathLoader {
  constructor (rootPath, ignoreVcsIgnores, traverseSymlinkDirectories, ignoredNames, useRipGrep) {
    this.rootPath = rootPath
    this.traverseSymlinkDirectories = traverseSymlinkDirectories
    this.ignoredNames = ignoredNames
    this.useRipGrep = useRipGrep
    this.paths = []
    this.inodes = new Set()
    this.repo = null
    if (ignoreVcsIgnores && !this.useRipGrep) {
      const repo = GitRepository.open(this.rootPath, {refreshOnWindowFocus: false})
      if ((repo && repo.relativize(path.join(this.rootPath, 'test'))) === 'test') {
        this.repo = repo
      }
    }
  }

  load (done) {
    if (this.useRipGrep) {
      this.loadFromRipGrep().then(done)

      return
    }

    this.loadPath(this.rootPath, true, () => {
      this.flushPaths()
      if (this.repo != null) this.repo.destroy()
      done()
    })
  }

  async loadFromRipGrep () {
    return new Promise((resolve) => {
      const args = ['--files', '--hidden', '--sort', 'path']

      if (this.ignoreVcsIgnores) {
        args.push('--no-ignore')
      }

      if (this.traverseSymlinkDirectories) {
        args.push('--follow')
      }

      for (let ignoredName of this.ignoredNames) {
        args.push('-g', '!' + ignoredName.pattern)
      }

      let output = ''
      const result = childProcess.spawn(realRgPath, args, {cwd: this.rootPath})

      result.stdout.on('data', chunk => {
        const files = (output + chunk).split('\n')
        output = files.pop()

        for (const file of files) {
          this.pathLoaded(path.join(this.rootPath, file))
        }
      })
      result.on('close', () => {
        this.flushPaths()
        resolve()
      })
    })
  }

  isIgnored (loadedPath) {
    const relativePath = path.relative(this.rootPath, loadedPath)
    if (this.repo && this.repo.isPathIgnored(relativePath)) {
      return true
    } else {
      for (let ignoredName of this.ignoredNames) {
        if (ignoredName.match(relativePath)) return true
      }
    }
  }

  pathLoaded (loadedPath, done) {
    if (!emittedPaths.has(loadedPath)) {
      this.paths.push(loadedPath)
      emittedPaths.add(loadedPath)
    }

    if (this.paths.length === PathsChunkSize) {
      this.flushPaths()
    }
    done && done()
  }

  flushPaths () {
    emit('load-paths:paths-found', this.paths)
    this.paths = []
  }

  loadPath (pathToLoad, root, done) {
    if (this.isIgnored(pathToLoad) && !root) return done()

    fs.lstat(pathToLoad, (error, stats) => {
      if (error != null) { return done() }
      if (stats.isSymbolicLink()) {
        fs.stat(pathToLoad, (error, stats) => {
          if (error != null) return done()
          if (this.inodes.has(stats.ino)) {
            return done()
          } else {
            this.inodes.add(stats.ino)
          }

          if (stats.isFile()) {
            this.pathLoaded(pathToLoad, done)
          } else if (stats.isDirectory()) {
            if (this.traverseSymlinkDirectories) {
              this.loadFolder(pathToLoad, done)
            } else {
              done()
            }
          } else {
            done()
          }
        })
      } else {
        this.inodes.add(stats.ino)
        if (stats.isDirectory()) {
          this.loadFolder(pathToLoad, done)
        } else if (stats.isFile()) {
          this.pathLoaded(pathToLoad, done)
        } else {
          done()
        }
      }
    })
  }

  loadFolder (folderPath, done) {
    fs.readdir(folderPath, (_, children = []) => {
      async.each(
        children,
        (childName, next) => {
          this.loadPath(path.join(folderPath, childName), false, next)
        },
        done
      )
    })
  }
}

module.exports = function (rootPaths, followSymlinks, ignoreVcsIgnores, ignores, useRipGrep) {
  const ignoredNames = []
  for (let ignore of ignores) {
    if (ignore) {
      try {
        ignoredNames.push(new Minimatch(ignore, {matchBase: true, dot: true}))
      } catch (error) {
        console.warn(`Error parsing ignore pattern (${ignore}): ${error.message}`)
      }
    }
  }

  async.eachLimit(
    rootPaths,
    MaxConcurrentCrawls,
    (rootPath, next) =>
      new PathLoader(
        rootPath,
        ignoreVcsIgnores,
        followSymlinks,
        ignoredNames,
        useRipGrep
      ).load(next)
    ,
    this.async()
  )
}