perf!: move build state into install state (#2574)

.gitignore

@@ -1,3 +1,12 @@
+# https://yarnpkg.com/getting-started/qa/#which-files-should-be-gitignored
+.yarn/*
+!.yarn/cache
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
 node_modules
 yarn-error.log
 
@@ -12,12 +21,6 @@ package.tgz
 /artifacts
 /dist
 
-# This is the Yarn build state; it's local to each clone
-/.yarn/build-state.yml
-
-# This is the Yarn install state cache, it can be rebuilt anytime
-/.yarn/install-state.gz
-
 # Our documentation is now handled by Netlify
 /docs
 
@@ -33,4 +36,4 @@ package.tgz
 # We check-in the PnP hook because it would be heavy to regenerate it before each bundle build
 !/packages/yarnpkg-pnp/lib/hook.js
 
-/vscode-case-study
+/vscode-case-study

.yarn/versions/aad4c985.yml

@@ -0,0 +1,30 @@
+releases:
+  "@yarnpkg/cli": major
+  "@yarnpkg/core": major
+  "@yarnpkg/plugin-essentials": major
+  "@yarnpkg/plugin-node-modules": major
+  "@yarnpkg/plugin-pack": major
+  "@yarnpkg/plugin-stage": major
+
+declined:
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/pnpify"

CHANGELOG.md

@@ -13,13 +13,15 @@
 - Yarn will now generate `.pnp.cjs` files (instead of `.pnp.js`) when using PnP, regardless of what the `type` field inside the manifest is set to.
 - The `-a` alias flag of `yarn workspaces foreach` got removed; use `-A,--all` instead, which is strictly the same.
 - The old PnPify SDK folder (`.vscode/pnpify`) won't be cleaned up anymore.
+- The `bstatePath` configuration option has been removed. The build state (`.yarn/build-state.yml`) has been moved into the install state (`.yarn/install-state.gz`)
 
 ### API
 
 - `structUtils.requirableIdent` got removed; use `structUtils.stringifyIdent` instead, which is strictly the same.
 - `configuration.format` got removed; use `formatUtils.pretty` instead, which is strictly the same, but type-safe.
 - `httpUtils.Options['json']` got removed; use `httpUtils.Options['jsonResponse']` instead, which is strictly the same.
 - `PackageExtension['description]` got removed, use `formatUtils.json(packageExtension, formatUtils.Type.PACKAGE_EXTENSION)` instead, which is strictly the same.
+- `Project.generateBuildStateFile` has been removed, the build state is now in `Project.storedBuildState`
 
 ### Bugfixes
 

packages/gatsby/content/features/zero-installs.md

@@ -31,7 +31,7 @@ In order to make a project zero-install, you must be able to use it as soon as y
 
 - When running `yarn install`, Yarn will generate a `.pnp.cjs` file. Add it to your repository as well - it contains the dependency tree that Node will use to load your packages.
 
-- Depending on whether your dependencies have install scripts or not (we advise you to avoid it if you can and prefer wasm-powered alternatives) you may also want to add the `.yarn/unplugged` and `.yarn/build-state.yml` entries.
+- Depending on whether your dependencies have install scripts or not (we advise you to avoid it if you can and prefer wasm-powered alternatives) you may also want to add the `.yarn/unplugged` entries.
 
 And that's it! Push your changes to your repository, checkout a new one somewhere, and check whether running `yarn start` (or whatever other script you'd normally use) works.
 

packages/gatsby/content/getting-started/questions-and-answers.md

@@ -75,7 +75,7 @@ If you're interested to know more about each of these files:
 
 - `.yarn/sdks` contains the editor SDKs generated by PnPify. Whether to keep it in your repository or not is up to you; if you don't, you'll need to follow the editor procedure again on new clones. See [Editor SDKs](/getting-started/editor-sdks) for more details.
 
-- `.yarn/unplugged` and `.yarn/build-state.yml` should likely always be ignored since they typically hold machine-specific build artifacts. Ignoring them might however prevent [Zero-Installs](https://yarnpkg.com/features/zero-installs) from working (to prevent this, set [`enableScripts`](/configuration/yarnrc#enableScripts) to `false`).
+- `.yarn/unplugged` should likely always be ignored since they typically hold machine-specific build artifacts. Ignoring it might however prevent [Zero-Installs](https://yarnpkg.com/features/zero-installs) from working (to prevent this, set [`enableScripts`](/configuration/yarnrc#enableScripts) to `false`).
 
 - `.yarn/versions` is used by the [version plugin](/features/release-workflow) to store the package release definitions. You will want to keep it within your repository.
 

packages/gatsby/static/configuration/yarnrc.json

@@ -19,13 +19,6 @@
   ],
   "type": "object",
   "properties": {
-    "bstatePath": {
-      "_package": "@yarnpkg/core",
-      "description": "This setting defines the location where the bstate file will be stored. The bstate file contains the current build state of each package that has build requirements in your dependencies. Removing the bstate file is safe, but will cause all your packages to be rebuilt.",
-      "type": "string",
-      "format": "uri-reference",
-      "default": "./.yarn/build-state.yml"
-    },
     "cacheFolder": {
       "_package": "@yarnpkg/core",
       "description": "The path where the downloaded packages are stored on your system. They'll be normalized, compressed, and saved under the form of zip archives with standardized names. The cache is deemed to be relatively safe to be shared by multiple projects, even when multiple Yarn instances run at the same time on different projects. For setting a global cache folder, you should use `enableGlobalCache` instead.",

packages/plugin-essentials/sources/commands/rebuild.ts

@@ -1,9 +1,7 @@
-import {BaseCommand, WorkspaceRequiredError}            from '@yarnpkg/cli';
-import {Cache, Configuration, IdentHash, StreamReport}  from '@yarnpkg/core';
-import {ThrowReport, structUtils, Project, LocatorHash} from '@yarnpkg/core';
-import {xfs, ppath}                                     from '@yarnpkg/fslib';
-import {parseSyml}                                      from '@yarnpkg/parsers';
-import {Command, Option, Usage}                         from 'clipanion';
+import {BaseCommand, WorkspaceRequiredError}           from '@yarnpkg/cli';
+import {Cache, Configuration, IdentHash, StreamReport} from '@yarnpkg/core';
+import {ThrowReport, structUtils, Project}             from '@yarnpkg/core';
+import {Command, Option, Usage}                        from 'clipanion';
 
 // eslint-disable-next-line arca/no-default-export
 export default class RunCommand extends BaseCommand {
@@ -43,39 +41,21 @@ export default class RunCommand extends BaseCommand {
     for (const identStr of this.idents)
       filteredIdents.add(structUtils.parseIdent(identStr).identHash);
 
+    await project.restoreInstallState();
+
     await project.resolveEverything({
       cache,
       report: new ThrowReport(),
     });
 
-    const bstatePath = configuration.get(`bstatePath`);
-    const bstate = xfs.existsSync(bstatePath)
-      ? parseSyml(await xfs.readFilePromise(bstatePath, `utf8`)) as {[key: string]: string}
-      : {};
-
-    const nextBState = new Map<LocatorHash, string>();
-
-    for (const pkg of project.storedPackages.values()) {
-      if (!Object.prototype.hasOwnProperty.call(bstate, pkg.locatorHash))
-        continue;
-
-      if (filteredIdents.size === 0 || filteredIdents.has(pkg.identHash))
-        continue;
-
-      const buildHash = bstate[pkg.locatorHash];
-      nextBState.set(pkg.locatorHash, buildHash);
-    }
-
-    if (nextBState.size > 0) {
-      const bstatePath = configuration.get(`bstatePath`);
-      const bstateFile = Project.generateBuildStateFile(nextBState, project.storedPackages);
-
-      await xfs.mkdirPromise(ppath.dirname(bstatePath), {recursive: true});
-      await xfs.changeFilePromise(bstatePath, bstateFile, {
-        automaticNewlines: true,
-      });
+    if (filteredIdents.size > 0) {
+      for (const pkg of project.storedPackages.values()) {
+        if (filteredIdents.has(pkg.identHash)) {
+          project.storedBuildState.delete(pkg.locatorHash);
+        }
+      }
     } else {
-      await xfs.removePromise(bstatePath);
+      project.storedBuildState.clear();
     }
 
     const installReport = await StreamReport.start({

packages/plugin-node-modules/sources/NodeModulesLinker.ts

@@ -194,9 +194,7 @@ class NodeModulesInstaller implements Installer {
 
     // Remove build state as well, to force rebuild of all the packages
     if (preinstallState === null) {
-      const bstatePath = this.opts.project.configuration.get(`bstatePath`);
-      if (await xfs.existsPromise(bstatePath))
-        await xfs.unlinkPromise(bstatePath);
+      this.opts.project.storedBuildState.clear();
 
       preinstallState = {locatorMap: new Map(), binSymlinks: new Map(), locationTree: new Map()};
     }

packages/plugin-pack/sources/packUtils.ts

@@ -179,7 +179,6 @@ export async function genPackList(workspace: Workspace) {
 
   maybeRejectPath(ppath.resolve(project.cwd, configuration.get(`lockfileFilename`)));
 
-  maybeRejectPath(configuration.get(`bstatePath`));
   maybeRejectPath(configuration.get(`cacheFolder`));
   maybeRejectPath(configuration.get(`globalFolder`));
   maybeRejectPath(configuration.get(`installStatePath`));

packages/plugin-stage/sources/commands/stage.ts

@@ -61,7 +61,6 @@ export default class StageCommand extends BaseCommand {
     const {driver, root} = await findDriver(project.cwd);
 
     const basePaths: Array<PortablePath | null> = [
-      configuration.get(`bstatePath`),
       configuration.get(`cacheFolder`),
       configuration.get(`globalFolder`),
       configuration.get(`virtualFolder`),

packages/yarnpkg-core/sources/Configuration.ts

@@ -180,11 +180,6 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} =
     type: SettingsType.ABSOLUTE_PATH,
     default: `./.yarn/$$virtual`,
   },
-  bstatePath: {
-    description: `Path of the file where the current state of the built packages must be stored`,
-    type: SettingsType.ABSOLUTE_PATH,
-    default: `./.yarn/build-state.yml`,
-  },
   lockfileFilename: {
     description: `Name of the files where the Yarn dependency tree entries must be stored`,
     type: SettingsType.STRING,
@@ -490,7 +485,6 @@ export interface ConfigurationValueMap {
   cacheFolder: PortablePath;
   compressionLevel: `mixed` | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
   virtualFolder: PortablePath;
-  bstatePath: PortablePath;
   lockfileFilename: Filename;
   installStatePath: PortablePath;
   immutablePatterns: Array<string>;

packages/yarnpkg-core/sources/Project.ts

@@ -125,6 +125,10 @@ const INSTALL_STATE_FIELDS = {
     `storedPackages`,
     `lockFileChecksum`,
   ] as const,
+
+  restoreBuildState: [
+    `storedBuildState`,
+  ] as const,
 };
 
 type RestoreInstallStateOpts = {
@@ -166,6 +170,7 @@ export class Project {
   public storedDescriptors: Map<DescriptorHash, Descriptor> = new Map();
   public storedPackages: Map<LocatorHash, Package> = new Map();
   public storedChecksums: Map<LocatorHash, string> = new Map();
+  public storedBuildState: Map<LocatorHash, string> = new Map();
 
   public accessibleLocators: Set<LocatorHash> = new Set();
   public originalPackages: Map<LocatorHash, Package> = new Map();
@@ -227,28 +232,6 @@ export class Project {
     throw new UsageError(`The nearest package directory (${formatUtils.pretty(configuration, packageCwd, formatUtils.Type.PATH)}) doesn't seem to be part of the project declared in ${formatUtils.pretty(configuration, project.cwd, formatUtils.Type.PATH)}.\n\n- If the project directory is right, it might be that you forgot to list ${formatUtils.pretty(configuration, ppath.relative(project.cwd, packageCwd), formatUtils.Type.PATH)} as a workspace.\n- If it isn't, it's likely because you have a yarn.lock or package.json file there, confusing the project root detection.`);
   }
 
-  static generateBuildStateFile(buildState: Map<LocatorHash, string>, locatorStore: Map<LocatorHash, Locator>) {
-    let bstateFile = `# Warning: This file is automatically generated. Removing it is fine, but will\n# cause all your builds to become invalidated.\n`;
-
-    const bstateData = [...buildState].map(([locatorHash, hash]) => {
-      const locator = locatorStore.get(locatorHash);
-
-      if (typeof locator === `undefined`)
-        throw new Error(`Assertion failed: The locator should have been registered`);
-
-      return [structUtils.stringifyLocator(locator), locator.locatorHash, hash];
-    });
-
-    for (const [locatorString, locatorHash, buildHash] of miscUtils.sortMap(bstateData, [d => d[0], d => d[1]])) {
-      bstateFile += `\n`;
-      bstateFile += `# ${locatorString}\n`;
-      bstateFile += `${JSON.stringify(locatorHash)}:\n`;
-      bstateFile += `  ${buildHash}\n`;
-    }
-
-    return bstateFile;
-  }
-
   constructor(projectCwd: PortablePath, {configuration}: {configuration: Configuration}) {
     this.configuration = configuration;
     this.cwd = projectCwd;
@@ -1177,11 +1160,6 @@ export class Project {
       return builder.digest(`hex`);
     };
 
-    const bstatePath: PortablePath = this.configuration.get(`bstatePath`);
-    const bstate = xfs.existsSync(bstatePath)
-      ? parseSyml(await xfs.readFilePromise(bstatePath, `utf8`)) as {[key: string]: string}
-      : {};
-
     // We reconstruct the build state from an empty object because we want to
     // remove the state from packages that got removed
     const nextBState = new Map<LocatorHash, string>();
@@ -1221,12 +1199,12 @@ export class Project {
         const buildHash = getBuildHash(pkg, buildInfo.buildLocations);
 
         // No need to rebuild the package if its hash didn't change
-        if (Object.prototype.hasOwnProperty.call(bstate, pkg.locatorHash) && bstate[pkg.locatorHash] === buildHash) {
+        if (this.storedBuildState.get(pkg.locatorHash) === buildHash) {
           nextBState.set(pkg.locatorHash, buildHash);
           continue;
         }
 
-        if (Object.prototype.hasOwnProperty.call(bstate, pkg.locatorHash))
+        if (this.storedBuildState.has(pkg.locatorHash))
           report.reportInfo(MessageName.MUST_REBUILD, `${structUtils.prettyLocator(this.configuration, pkg)} must be rebuilt because its dependency tree changed`);
         else
           report.reportInfo(MessageName.MUST_BUILD, `${structUtils.prettyLocator(this.configuration, pkg)} must be built because it never did before or the last one failed`);
@@ -1319,21 +1297,11 @@ export class Project {
       }
     }
 
-    // We can now generate the bstate file, which will allow us to "remember"
+    // We can now update the storedBuildState, which will allow us to "remember"
     // what's the dependency tree subset that we used to build a specific
     // package (and avoid rebuilding it later if it didn't change).
 
-    if (nextBState.size > 0) {
-      const bstatePath = this.configuration.get(`bstatePath`);
-      const bstateFile = Project.generateBuildStateFile(nextBState, this.storedPackages);
-
-      await xfs.mkdirPromise(ppath.dirname(bstatePath), {recursive: true});
-      await xfs.changeFilePromise(bstatePath, bstateFile, {
-        automaticNewlines: true,
-      });
-    } else {
-      await xfs.removePromise(bstatePath);
-    }
+    this.storedBuildState = nextBState;
   }
 
   async install(opts: InstallOptions) {
@@ -1596,7 +1564,7 @@ export class Project {
     await xfs.changeFilePromise(installStatePath, serializedState as Buffer);
   }
 
-  async restoreInstallState({restoreInstallersCustomData = true, restoreResolutions = true}: RestoreInstallStateOpts = {}) {
+  async restoreInstallState({restoreInstallersCustomData = true, restoreResolutions = true, restoreBuildState = true}: RestoreInstallStateOpts = {}) {
     const installStatePath = this.configuration.get(`installStatePath`);
     if (!xfs.existsSync(installStatePath)) {
       if (restoreResolutions)
@@ -1619,6 +1587,10 @@ export class Project {
         await this.applyLightResolution();
       }
     }
+
+    if (restoreBuildState) {
+      Object.assign(this, pick(installState, INSTALL_STATE_FIELDS.restoreBuildState));
+    }
   }
 
   async applyLightResolution() {