Merge d6b2d6f into 7d30246

.gitignore

@@ -1,7 +1,4 @@
 /node_modules
 /coverage
 *.iml
-!/test/fixtures/project1/node_modules/
-!/test/fixtures/moduleWithDeps/node_modules/
-!/test/fixtures/moduleWithDeps/node_modules/dep2/node_modules/
-!/test/fixtures/moduleWithDeps/dir/node_modules/
+/.idea

.thought/partials/howitworks.md.hbs

@@ -0,0 +1,42 @@
+# How it works
+
+`{{packageJson.name}}` collects all modules from the `node_modules` directory, the `node_modules` directory of 
+each of those modules and the `node_modules` directory in that modules, and so on.
+
+When all packages have been collected, it reads the package.json of each module and uses the `_location`-property
+and the `_requiredBy`-property to recreate the complete dependency tree.
+
+* `_location` contains the location of the module in the directory tree. A module in `node_modules/packageA/node_modules/packageB`
+  has the location `/packageA/packageB`
+* `_requiredBy` contains a list of module that are dependent on the current module. For each such module, it contains
+  the value of the `_location`-property.
+
+Once the packages is connected, the stats for each package are computed:
+
+* The number of dependencies is computed transitively across the tree.
+* The total kilobytes (1024 bytes) is computed, include all dependencies.
+  The computation of file sizes assumes that only whole blocks are used, even by small files. The `blksize`-property
+  of the [fs.Stats-object]() is used as block size. If this value is missing (e.g. on Windows), a size of 4096 is 
+  used.
+
+## Caveats
+
+In some cases, the dependencies in the `node_modules`-directory are tempered with. For example, {{npm 'lerna'}}
+combines dependencies of multiple packages in the `node_modules`-directory of the root-project and removes 
+obsolete dependencies from the tree. This can lead to cycles in the dependency tree which are displayed in
+the output like this:
+
+{{include 'test/fixtures/moduleWithCyclicDeps.txt'}}
+
+Furthermore, this and the use of optional dependencies can lead to a situation where a package is `_requiredBy` 
+an existing dependency (i.e. a dependent package) but does not exist anymore in the tree. For those delete packages,
+a dummy package is displayed in a separate tree.
+
+{{include 'test/fixtures/moduleWithMissingDependent.txt'}}
+
+In  this example, a module `dep2@1.0.0` was found. The `_requireBy`-property shows that `dep2`
+is part of the tree, because it is a dependency of a module that should be in `node_modules/dep3`, 
+which could not be found.
+
+
+

README.md

@@ -29,7 +29,7 @@ Run `analyze-module-size` in your project directory. The output will be somethin
 (Note that the displayed sizes are accumulated from the each module an its dependencies):
 
 ```
-size: 64k... with-dependencies: 1200k
+size: 68k... with-dependencies: 1204k
 ├─┬ globby@6.1.0, 488k, 17 deps
 │ ├─┬ glob@7.1.2, 344k, 10 deps
 │ │ ├─┬ minimatch@3.0.4, 136k, 3 deps
@@ -101,6 +101,63 @@ Usage: analyze-module-size [options]
 ```
 
 
+# How it works
+
+`` collects all modules from the `node_modules` directory, the `node_modules` directory of 
+each of those modules and the `node_modules` directory in that modules, and so on.
+
+When all packages have been collected, it reads the package.json of each module and uses the `_location`-property
+and the `_requiredBy`-property to recreate the complete dependency tree.
+
+* `_location` contains the location of the module in the directory tree. A module in `node_modules/packageA/node_modules/packageB`
+  has the location `/packageA/packageB`
+* `_requiredBy` contains a list of module that are dependent on the current module. For each such module, it contains
+  the value of the `_location`-property.
+
+Once the packages is connected, the stats for each package are computed:
+
+* The number of dependencies is computed transitively across the tree.
+* The total kilobytes (1024 bytes) is computed, include all dependencies.
+  The computation of file sizes assumes that only whole blocks are used, even by small files. The `blksize`-property
+  of the [fs.Stats-object]() is used as block size. If this value is missing (e.g. on Windows), a size of 4096 is 
+  used.
+
+## Caveats
+
+In some cases, the dependencies in the `node_modules`-directory are tempered with. For example, [lerna](https://npmjs.com/package/lerna)
+combines dependencies of multiple packages in the `node_modules`-directory of the root-project and removes 
+obsolete dependencies from the tree. This can lead to cycles in the dependency tree which are displayed in
+the output like this:
+
+```txt
+size: 42k... with-dependencies: 42k
+└─┬ dep1@1.0.0, 42k, 3 deps
+  └─┬ dep1a@1.0.0, 42k, 3 deps
+    └─┬ dep2@1.0.0, 42k, 3 deps
+      └── dep1@1.0.0 (cycle detected)
+
+```
+
+
+Furthermore, this and the use of optional dependencies can lead to a situation where a package is `_requiredBy` 
+an existing dependency (i.e. a dependent package) but does not exist anymore in the tree. For those delete packages,
+a dummy package is displayed in a separate tree.
+
+```txt
+size: 42k... with-dependencies: 42k
+
+missing packages, that are referenced as dependent of an existing dependency
+└─┬ /dep3, 42k, 1 deps
+  └── dep2@1.0.0, 42k, 0 deps
+
+```
+
+
+In  this example, a module `node_modules/dep2` was found. The `_requireBy`-property shows that `node_modules/dep2`
+is part of the tree, because it is a dependency of `node_modules/dep3`, which could not be found.
+
+
+
 
 # License
 

src/DependencyTree.js

@@ -8,11 +8,12 @@ const {NullProgressHandler} = require('./progress')
 class DependencyTree {
   /**
    * @param {Package} prod package containing the production dependencies (and the files in the repository itself)
-   * @param {Package} dev package containing the dev-dependencies
-   * @param {Package} manual package containing the manually installed dependencies
+   * @param {Package[]} dev packages in the devDependencies
+   * @param {Package[]} manual manually installed packages (non-dependencies)
+   * @param {Package[]} missing packages whose dependent could not be found
    * @param {Package[]} all all packages in a node_modules directory, in a flat list.
    */
-  constructor (rootPackage, prod, dev, manual, all) {
+  constructor (rootPackage, prod, dev, manual, missing, all) {
     this.rootPackage = rootPackage
     /**
      * @type {Package[]}
@@ -26,6 +27,10 @@ class DependencyTree {
      * @type {Package[]}
      */
     this.manual = manual
+    /**
+     * @type {Package[]}
+     */
+    this.missing = missing
     /**
      * @type {Package[]}
      */
@@ -60,13 +65,14 @@ class DependencyTree {
       })
     return deep({rootPackage, dependencies}).then(function ({rootPackage, dependencies}) {
       progressHandler.connectAll()
-      var {prod, dev, manual} = Package.connectAll(rootPackage, dependencies)
+      var {prod, dev, manual, missing} = Package.connectAll(rootPackage, dependencies)
       progressHandler.done()
       return new DependencyTree(
         rootPackage,
-        prod.dependencies,
-        dev.dependencies,
-        manual.dependencies,
+        prod,
+        dev,
+        manual,
+        missing,
         dependencies
       )
     })

src/Package.js

@@ -33,15 +33,12 @@ class Package {
    * Create references to dependencies and dependents
    * contained in the packages-map.
    *
-   * @param {object<Package>} packages packages as returned by Package#indexByLocation
+   * @param {object<Package>} packageIndex packages as returned by Package#indexByLocation
    */
-  connect (packages) {
+  connect (packageIndex) {
     if (this.packageJson._requiredBy) {
       this.packageJson._requiredBy.forEach((key) => {
-        const dependent = packages.get(key)
-        if (!dependent) {
-          throw new Error(`Could not find "${key}" in ${JSON.stringify(Array.from(packages.keys()))}`)
-        }
+        const dependent = packageIndex.get(key) || Package.dummyForMissingDependent(key, packageIndex)
         this.dependents.push(dependent)
         dependent.dependencies.push(this)
       })
@@ -96,26 +93,46 @@ class Package {
    */
   static indexByLocation (packages) {
     const map = new Map(packages.map(p => [p.location(), p]))
-    map.set('#USER', new Package())
-    map.set('#DEV:/', new Package())
+    map.set('#USER', new Package({id: '#USER'}))
+    map.set('#DEV:/', new Package({id: '#DEV:/'}))
+    map.set('#MISSING', new Package({id: "#MISSING'"}))
     return map
   }
 
   /**
-   *
-   * @param {...(Package|Package[])} packages
+   * Create a dummy package for a missing dependent.
+   * @param location the location of the dummy package (i.e. the entry in the _requiredBy-tag of the dependency
+   *   of this package)
+   * @param packageIndex the map of all packages (location -> package)
+   */
+  static dummyForMissingDependent (location, packageIndex) {
+    const dummy = new Package({_id: location, _location: location}, new PackageStats(null, []))
+    const sparePackage = packageIndex.get('#MISSING')
+    packageIndex.set(location, dummy)
+    dummy.dependents.push(sparePackage)
+    sparePackage.dependencies.push(dummy)
+    return dummy
+  }
+
+  /**
+   * Connect a like of packages. The list may contain packages and arrays of packages. It will be flattened
+   * @param {(Package|Package[])[]} packages the list of packages or list of packages
+   * @return {object} an object containing dependencies of the following kind:
+   *   prod ("dependencies"), dev ("devDependencies"), manual ("manually installed by the user"),
+   *   missing ("implicitly found as dependent of another package")
    */
   static connectAll (...packages) {
     debug('Connect all')
     // flatten
     const flatPackages = Array.prototype.concat.apply([], packages)
-    const index = Package.indexByLocation(flatPackages)
-    flatPackages.forEach((pkg) => pkg.connect(index))
+    const packageIndex = Package.indexByLocation(flatPackages)
+    flatPackages.forEach((pkg) => pkg.connect(packageIndex))
     debug('Done connect all')
     return {
-      prod: index.get('/'),
-      dev: index.get('#DEV:/'),
-      manual: index.get('#USER')
+      prod: packageIndex.get('/').dependencies,
+      dev: packageIndex.get('#DEV:/').dependencies,
+      manual: packageIndex.get('#USER').dependencies,
+      missing: packageIndex.get('#MISSING').dependencies
     }
   }
 }

src/PackageStats.js

@@ -14,15 +14,15 @@ class PackageStats {
   /**
    * Create a new PackageStats object
    * @param {string} directory path to the package-json files
-   * @param {{file: string, stat: fs.Stats}} files files and stats in this package
+   * @param {{file: string, stat: fs.Stats}[]} files files and stats in this package
    */
   constructor (directory, files) {
     /**
      * @type {string}
      */
     this.directory = directory
     /**
-     * @type {{file: string, stat: fs.Stats}}
+     * @type {{file: string, stat: fs.Stats}[]}
      */
     this.files = files
   }

src/index.js

@@ -21,27 +21,48 @@ const {NullProgressHandler} = require('./progress')
 function analyze (cwd, options = {}) {
   return DependencyTree.loadFrom(path.join(cwd, 'package.json'), options.progress || new NullProgressHandler())
     .then(function (tree) {
-      return archy({
+      let result = archy({
         label: `size: ${tree.rootPackage.stats.totalBlockSize() / 1024}k... with-dependencies: ${tree.rootPackage.totalStats().totalBlockSize() / 1024}k`,
-        nodes: toArchy(tree.prod, options.depth)
+        nodes: toArchy(tree.prod, options.depth, [])
       })
+      if (tree.missing.length > 0) {
+        result += '\n' + archy({
+          label: `missing packages, that are referenced as dependent of an existing dependency`,
+          nodes: toArchy(tree.missing, options.depth, [])
+        })
+      }
+      return result
     })
 }
 
 /**
+ * Create an archy-compatible object-structure of the dependency tree.
  *
  * @param pkgs
  * @param {number=} depth
+ * @param {string[]} cycleChecker list of "_location"s on the current path down the dependency tree.
  */
-function toArchy (pkgs, depth) {
+function toArchy (pkgs, depth, cycleChecker) {
   if (depth <= 0) return []
   const result = pkgs.map(pkg => {
-    const blockSize = pkg.totalStats().totalBlockSize()
-    const dependencyCount = pkg.totalDependencies()
-    return {
-      label: `${pkg.packageJson._id}, ${chalk.red(blockSize / 1024 + 'k')}, ${dependencyCount} deps`,
-      size: blockSize,
-      nodes: toArchy(pkg.dependencies, depth && depth - 1)
+    if (cycleChecker.indexOf(pkg.packageJson._location) >= 0) {
+      return {
+        label: `${pkg.packageJson._id} (cycle detected)`,
+        size: undefined,
+        nodes: []
+      }
+    }
+    cycleChecker.push(pkg.packageJson._location)
+    try {
+      const blockSize = pkg.totalStats().totalBlockSize()
+      const dependencyCount = pkg.totalDependencies()
+      return {
+        label: `${pkg.packageJson._id}, ${chalk.red(blockSize / 1024 + 'k')}, ${dependencyCount} deps`,
+        size: blockSize,
+        nodes: toArchy(pkg.dependencies, depth && depth - 1, cycleChecker)
+      }
+    } finally {
+      cycleChecker.pop()
     }
   })
   return sortby(result, (node) => {

src/progress.js

@@ -4,7 +4,7 @@ const ProgressBar = require('progress')
 class ProgressHandler {
   constructor (stream) {
     this.stream = stream
-    this.foundDepsProgress = new ProgressBar('dependencies found: :current', {total: 1000, stream: this.stream})
+    this.foundDepsProgress = new ProgressBar('dependencies found: :current', {total: 1000000, stream: this.stream})
     this.loadedDepsProgress = null
   }