Pure store (#524)

* feat: drop Node.js <= 6.3 support

BREAKING CHANGE:

drop support of Node.js pre-6.3

* refactor: remove the redundant subfolder in store

* refactor: rename linkPeers

* refactor: remove not used code

* feat: link each file separately

* refactor: change store structure

* refactor: resolve github dependencies to commits

* chore(package.json): pass --preserve-symlinks to ts-node

* test: temporarily skip failing tests

* refactor(git): use more human-friendly Git IDs

* refactor: resolve

* refactor: divide resolution and fetching

* refactor(fetch): better variable naming

* feat: add shrinkwrap.yaml

* test(shrinkwrap): shrinkwrap locks dependencies

* style: fix typing issue

* fix(fetch): properly link node_modules on repeat install

* refactor: comment skipped test with single-line comments

* fix: circular symlinks are avoided

* refactor(fetch): remove redundant type properties

* test: additional check for circular symlinks

* feat: saving dependency graph in node_modules instead of in store

BREAKING CHANGE:

Dependency graph moved out from store

* feat: add store.yaml with info about dependent projects

* refactor: remove name from FetchedPackage

* refactor: change the location of packages from npm in store

BREAKING CHANGE:

Structure of store changed

* refactor: change global store path

BREAKING CHANGE:

Move the store path out of ~/.pnpm

* refactor: remove justFetched from FetchedPackage

* refactor(install): use functional programming to filter deps

* refactor(install): remove redundant node_modules creation

* fix(install): installation of concurent circular dependencies

* feat: copy not link some packages

Packages that have install lifecycle events are copied not linked.

* fix(store.yaml): don't duplicate records

* perf: resolve a package only once per project

* test: concurrent installation of the same dependency

* fix: packages are not removed in the middle of fetch

* fix: dependencies are linked only once

* fix: make linking work on Windows

* fix: installing local dependencies on Windows

* fix: don't reinstall dependencies of the same package

* fix(bin): the run function always returns a Promise

* fix: use hard links to link files to the .resolutions folder

* fix: peers are linked into the correct location

* fix: bundled dependencies are not reinstalled

* refactor: remove unsymlink

It is not used anymore

* chore(CI): test on Node.js v4 as well

* refactor: use UPPER_CASE for constants

* chore(package): return support of Node.js v4

* fix: make pnpm Node.js 4 compatible again

* test: don't run tsnode with --preserve-symlinks

--preserve-symlinks makes tests fail on Node.js 4

* fix(bin): don't use --preserve-symlinks on Node pre-6.3

* test: use global stores in tests

* fix: flat-tree installation should not work on Node 4

* fix: show warning

When trying to install into a node_modules created by older pnpm

* refactor: change the global store location

From `~/.store` to `~/.pnpm-store`

* docs: note that some of the info is out of date

* test: fix typing error

* test: fix frequently failing test

README.md

@@ -27,21 +27,40 @@ Follow the [pnpm Twitter account](https://twitter.com/pnpmjs) for updates.
 
 ## Background
 
-`pnpm` maintains a flat storage of all your dependencies in `node_modules/.store`. They are then symlinked wherever they're needed.
+`pnpm` maintains a flat storage of all your dependencies in `~/.pnpm-store`. They are then linked wherever they're needed.
 This nets you the benefits of less disk space usage, while keeping your `node_modules` clean.
 See [store layout](docs/store-layout.md) for an explanation.
 
 ```
+=> - a link (also known as a hard link)
+-> - a symlink (or junction on Windows)
+
+~/.store
+   ├─ chalk/1.1.1/
+   |  ├─ index.js
+   |  └─ package.json
+   ├─ ansi-styles/2.1.0/
+   |  ├─ index.js
+   |  └─ package.json
+   └─ has-ansi/2.0.0/
+      ├─ index.js
+      └─ package.json
 .
 └─ node_modules/
-   ├─ .store/
-   │  ├─ chalk@1.1.1/_/
-   │  │  └─ node_modules/
-   │  │     ├─ ansi-styles      -> ../../../ansi-styles@2.1.0/_
-   │  │     └─ has-ansi         -> ../../../has-ansi@2.0.0/_
-   │  ├─ ansi-styles@2.1.0/_/
-   │  └─ has-ansi@2.0.0/_/
-   └─ chalk                     -> .store/chalk@1.1.1/_
+   ├─ .resolutions/
+   |   ├─ chalk/1.1.1/
+   |   |  ├─ node_modules/
+   |   |  |  ├─ ansi-styles/ -> ../../ansi-styles/2.1.0/
+   |   |  |  └─ has-ansi/    -> ../../has-ansi/2.0.0/
+   |   |  ├─ index.js        => ~/.store/chalk/1.1.1/index.js
+   |   |  └─ package.json    => ~/.store/chalk/1.1.1/package.json
+   |   ├─ has-ansi/2.0.0/
+   |   |  ├─ index.js        => ~/.store/has-ansi/2.0.0/index.js
+   |   |  └─ package.js      => ~/.store/has-ansi/2.0.0/package.json
+   |   └─ ansi-styles/2.1.0/
+   |      ├─ index.js        => ~/.store/ansi-styles/2.1.0/index.js
+   |      └─ package.js      => ~/.store/ansi-styles/2.1.0/package.json
+   └─ chalk/                 -> ./.resolutions/chalk/1.1.1/
 ```
 
 ## Install

docs/store-layout.md

@@ -1,5 +1,7 @@
 # Store layout
 
+## This documentation is currently out of date!
+
 `pnpm` maintains a flat storage of all your dependencies in `node_modules/.store`. They are then symlinked whereever they're needed.
 This is like `npm@2`'s recursive module handling (without the disk space bloat), and like `npm@3`s flat dependency tree (except with each module being predictably atomic).
 To illustrate, an installation of [chalk][]@1.1.1 may look like this:

docs/store-yaml.md

@@ -1,5 +1,7 @@
 # store.yaml
 
+## This documentation is currently out of date!
+
 `store.yaml` contains information about all the different internal/external dependencies that the packages in the store have. This is especially useful because `pnpm` allows to use shared stores.
 
 ## pnpm

sinopia/storage/dep-of-pkg-with-1-dep/package.json

@@ -63,8 +63,9 @@
 	},
 	"dist-tags": {
 		"latest": [
-			"1.1.0",
-			"100.0.0",
+			"100.1.0"
+		],
+		"stable": [
 			"100.1.0"
 		]
 	},
@@ -91,6 +92,6 @@
 			"fetched": 1482541753493
 		}
 	},
-	"_rev": "7-9c25707ca43f0fd2",
+	"_rev": "27-82b91977c3031e63",
 	"readme": "ERROR: No README data found!"
 }

src/api/cache.ts

@@ -1,10 +1,10 @@
 import path = require('path')
 import rimraf = require('rimraf-then')
-import {GlobalPath as DefaultGlobalPath} from './constantDefaults'
+import {DEFAULT_GLOBAL_PATH} from './constantDefaults'
 import expandTilde from '../fs/expandTilde'
 
 export function cleanCache (globalPath?: string) {
-  globalPath = globalPath || DefaultGlobalPath
+  globalPath = globalPath || DEFAULT_GLOBAL_PATH
   const cachePath = getCachePath(globalPath)
   return rimraf(cachePath)
 }

src/api/constantDefaults.ts

@@ -1,2 +1,2 @@
-export const GlobalPath = '~/.pnpm'
-export const GlobalStorePath = GlobalPath + '/.store'
+export const DEFAULT_GLOBAL_PATH = '~/.pnpm'
+export const DEFAULT_GLOBAL_STORE_PATH = '~/.pnpm-store'

src/api/extendOptions.ts

@@ -1,5 +1,5 @@
 import {StrictPnpmOptions, PnpmOptions} from '../types'
-import {GlobalPath as globalPath, GlobalStorePath} from './constantDefaults'
+import {DEFAULT_GLOBAL_PATH, DEFAULT_GLOBAL_STORE_PATH} from './constantDefaults'
 import {preserveSymlinks} from '../env'
 import {LoggerType} from '../logger' // tslint:disable-line
 
@@ -8,8 +8,8 @@ const defaults = () => (<StrictPnpmOptions>{
   fetchRetryFactor: 10,
   fetchRetryMintimeout: 1e4, // 10 seconds
   fetchRetryMaxtimeout: 6e4, // 1 minute
-  storePath: getDefaultStorePath(),
-  globalPath,
+  storePath: DEFAULT_GLOBAL_STORE_PATH,
+  globalPath: DEFAULT_GLOBAL_PATH,
   logger: 'pretty',
   ignoreScripts: false,
   linkLocal: false,
@@ -26,11 +26,6 @@ const defaults = () => (<StrictPnpmOptions>{
   engineStrict: false,
 })
 
-function getDefaultStorePath () {
-  if (preserveSymlinks) return GlobalStorePath
-  return 'node_modules/.store'
-}
-
 export default (opts?: PnpmOptions): StrictPnpmOptions => {
   opts = opts || {}
   if (opts.flatTree === true && !preserveSymlinks) {

src/api/getContext.ts

@@ -8,27 +8,30 @@ import expandTilde, {isHomepath} from '../fs/expandTilde'
 import {StrictPnpmOptions} from '../types'
 import initLogger from '../logger'
 import {
-  read as readStore,
-  create as createStore,
-  Store,
-  TreeType,
-} from '../fs/storeController'
+  read as readGraph,
+  Graph,
+} from '../fs/graphController'
+import {
+  read as readShrinkwrap,
+  Shrinkwrap,
+} from '../fs/shrinkwrap'
 import {
-  read as readModules
+  read as readModules,
+  TreeType,
 } from '../fs/modulesController'
 import mkdirp from '../fs/mkdirp'
 import {Package} from '../types'
 import {getCachePath} from './cache'
 import normalizePath = require('normalize-path')
-import {preserveSymlinks} from '../env'
-import {GlobalStorePath} from './constantDefaults'
+import {DEFAULT_GLOBAL_STORE_PATH} from './constantDefaults'
 
 export type PnpmContext = {
   pkg?: Package,
   cache: string,
   storePath: string,
   root: string,
-  store: Store,
+  graph: Graph,
+  shrinkwrap: Shrinkwrap,
   isFirstInstallation: boolean,
 }
 
@@ -37,13 +40,8 @@ export default async function (opts: StrictPnpmOptions): Promise<PnpmContext> {
   const root = normalizePath(pkg.path ? path.dirname(pkg.path) : opts.cwd)
   const storeBasePath = resolveStoreBasePath(opts.storePath, root)
 
-  // to avoid orphan packages created with pnpm v0.41.0 and earlier
-  if (!underNodeModules(storeBasePath) && await readStore(storeBasePath)) {
-    throw new Error(structureChangeMsg('Shared stores were divided into types, flat and nested. https://github.com/rstacruz/pnpm/pull/429'))
-  }
-
   const treeType: TreeType = opts.flatTree ? 'flat' : 'nested'
-  const storePath = getStorePath(treeType, storeBasePath)
+  const storePath = getStorePath(storeBasePath)
 
   let modules = await readModules(path.join(root, 'node_modules'))
   const isFirstInstallation: boolean = !modules
@@ -52,28 +50,31 @@ export default async function (opts: StrictPnpmOptions): Promise<PnpmContext> {
     err['code'] = 'ALIEN_STORE'
     throw err
   }
-
-  const store = await readStore(storePath) || createStore(treeType)
-  store.type = store.type || 'nested' // for backward compatibility with v0.41.0 and earlier
-  if (store.type !== treeType) {
-    const err = new Error(`Cannot use a ${store.type} store for a ${treeType} installation`)
+  if (modules && modules.type !== treeType) {
+    const err = new Error(`Cannot use a ${modules.type} store for a ${treeType} installation`)
     err['code'] = 'INCONSISTENT_TREE_TYPE'
     throw err
   }
-  if (store.preserveSymlinks !== preserveSymlinks) {
-    const err = new Error(`Cannot use a store installed with preserveSymlinks = ${store.preserveSymlinks}. Need store with preserveSymlinks = ${preserveSymlinks}`)
-    err['code'] = 'INCONSISTENT_PRESERVE_SYMLINKS'
-    throw err
-  }
-  if (store) {
-    failIfNotCompatible(store.pnpm)
+  if (modules) {
+    if (!modules.packageManager) {
+      const msg = structureChangeMsg(stripIndent`
+        The change was needed to allow machine stores and dependency locks:
+          PR: https://github.com/rstacruz/pnpm/pull/524
+      `)
+      throw new Error(msg)
+    }
+    failIfNotCompatible(modules.packageManager.split('@')[1])
   }
+
+  const graph = await readGraph(path.join(root, 'node_modules')) || {}
+  const shrinkwrap = await readShrinkwrap(root) || {}
   const ctx: PnpmContext = {
     pkg: pkg.pkg,
     root,
     cache: getCachePath(opts.globalPath),
     storePath,
-    store,
+    graph,
+    shrinkwrap,
     isFirstInstallation,
   }
 
@@ -139,7 +140,7 @@ const DefaultGlobalPkg: Package = {
   private: true,
   config: {
     npm: {
-      storePath: GlobalStorePath
+      storePath: DEFAULT_GLOBAL_STORE_PATH
     }
   }
 }
@@ -162,16 +163,11 @@ function resolveStoreBasePath (storePath: string, pkgRoot: string) {
   return path.resolve(pkgRoot, storePath)
 }
 
-function getStorePath (treeType: TreeType, storeBasePath: string): string {
+function getStorePath (storeBasePath: string): string {
   if (underNodeModules(storeBasePath)) {
     return storeBasePath
   }
-  // potentially shared stores have to have separate subdirs for different
-  // installation types
-  if (preserveSymlinks) {
-    return path.join(storeBasePath, treeType)
-  }
-  return path.join(storeBasePath, `${treeType}-node.v4`)
+  return path.join(storeBasePath, '1')
 }
 
 function underNodeModules (dirpath: string): boolean {