Selective build (#2202)

* Accept --changed and --compareBranch options

* Implement selective build of changed workspaces

* Remove TODO comment

* Exit process from main build function if no targets

* Add fixtures and first test

* Test only --changed

* Allow package names with --changed

* Remove lockfile in fixtures

* Enable --descendants

* Remove tests from fixtures

* Test order of builds

* Introduce ancestors, factor out selectWorkspaces and fix build order

* Fix tests and docs

* Create real-eyes-vanish.md

* Better wording and tsdoc

* Fix wording

.changeset/real-eyes-vanish.md

@@ -0,0 +1,5 @@
+---
+"modular-scripts": minor
+---
+
+Selective build support

__fixtures__/ghost-building/.editorconfig

@@ -0,0 +1,18 @@
+# https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+max_line_length = 80
+trim_trailing_whitespace = true
+
+[*.{md,mdx}]
+max_line_length = 0
+trim_trailing_whitespace = false
+
+[COMMIT_EDITMSG]
+max_line_length = 0

__fixtures__/ghost-building/.eslintignore

@@ -0,0 +1,23 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+packages/**/public
+/dist

__fixtures__/ghost-building/.gitignore

@@ -0,0 +1,24 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+/dist
+
+.vscode

__fixtures__/ghost-building/.prettierignore

@@ -0,0 +1,2 @@
+/dist
+/packages/**/public

__fixtures__/ghost-building/.yarnrc

@@ -0,0 +1 @@
+disable-self-update-check true

__fixtures__/ghost-building/README.md

@@ -0,0 +1 @@
+This is the `README.md` for the whole monorepo.

__fixtures__/ghost-building/modular/setupEnvironment.ts

@@ -0,0 +1,2 @@
+// Allows for adding setup configuration to Jest
+export {};

__fixtures__/ghost-building/modular/setupTests.ts

@@ -0,0 +1,5 @@
+// jest-dom adds custom jest matchers for asserting on DOM nodes.
+// allows you to do things like:
+// expect(element).toHaveTextContent(/react/i)
+// learn more: https://github.com/testing-library/jest-dom
+import '@testing-library/jest-dom/extend-expect';

__fixtures__/ghost-building/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "ghost-building",
+  "version": "1.0.0",
+  "main": "index.js",
+  "author": "Cristiano Belloni <cristiano.belloni@jpmorgan.com>",
+  "license": "MIT",
+  "private": true,
+  "workspaces": [
+    "packages/**"
+  ],
+  "modular": {
+    "type": "root"
+  },
+  "scripts": {
+    "start": "modular start",
+    "build": "modular build",
+    "test": "modular test",
+    "lint": "eslint . --ext .js,.ts,.tsx",
+    "prettier": "prettier --write ."
+  },
+  "eslintConfig": {
+    "extends": "modular-app"
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "prettier": {
+    "singleQuote": true,
+    "trailingComma": "all",
+    "printWidth": 80,
+    "proseWrap": "always"
+  },
+  "dependencies": {
+    "@testing-library/dom": "^8.19.0",
+    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/react": "^13.4.0",
+    "@testing-library/user-event": "^7.2.1",
+    "@types/jest": "^29.2.3",
+    "@types/node": "^18.11.9",
+    "@types/react": "^18.0.25",
+    "@types/react-dom": "^18.0.9",
+    "eslint-config-modular-app": "^3.0.2",
+    "modular-scripts": "^3.5.0",
+    "modular-template-app": "^1.1.0",
+    "modular-template-package": "^1.1.0",
+    "prettier": "^2.7.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "typescript": ">=4.2.1 <4.5.0"
+  }
+}

__fixtures__/ghost-building/packages/a/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "a",
+  "private": false,
+  "modular": {
+    "type": "package"
+  },
+  "main": "./src/index.ts",
+  "version": "1.0.0",
+  "dependencies": {
+    "b": "1.0.0",
+    "c": "1.0.0"
+  }
+}

__fixtures__/ghost-building/packages/a/src/index.ts

@@ -0,0 +1,3 @@
+export default function add(a: number, b: number): number {
+  return a + b;
+}

__fixtures__/ghost-building/packages/b/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "b",
+  "private": false,
+  "modular": {
+    "type": "package"
+  },
+  "main": "./src/index.ts",
+  "version": "1.0.0",
+  "dependencies": {
+    "c": "1.0.0"
+  }
+}

__fixtures__/ghost-building/packages/b/src/index.ts

@@ -0,0 +1,3 @@
+export default function add(a: number, b: number): number {
+  return a + b;
+}

__fixtures__/ghost-building/packages/c/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "c",
+  "private": false,
+  "modular": {
+    "type": "package"
+  },
+  "main": "./src/index.ts",
+  "version": "1.0.0",
+  "dependencies": {
+    "d": "1.0.0"
+  }
+}

__fixtures__/ghost-building/packages/c/src/index.ts

@@ -0,0 +1,3 @@
+export default function add(a: number, b: number): number {
+  return a + b;
+}

__fixtures__/ghost-building/packages/d/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "d",
+  "private": false,
+  "modular": {
+    "type": "package"
+  },
+  "main": "./src/index.ts",
+  "version": "1.0.0"
+}

__fixtures__/ghost-building/packages/d/src/index.ts

@@ -0,0 +1,3 @@
+export default function add(a: number, b: number): number {
+  return a + b;
+}

__fixtures__/ghost-building/packages/e/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "e",
+  "private": false,
+  "modular": {
+    "type": "package"
+  },
+  "main": "./src/index.ts",
+  "version": "1.0.0",
+  "dependencies": {
+    "a": "1.0.0"
+  }
+}

__fixtures__/ghost-building/packages/e/src/index.ts

@@ -0,0 +1,3 @@
+export default function add(a: number, b: number): number {
+  return a + b;
+}

__fixtures__/ghost-building/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "modular-scripts/tsconfig.json",
+  "include": ["modular", "packages/**/src"]
+}

docs/commands/build.md

@@ -3,10 +3,11 @@ parent: Commands
 title: modular build
 ---
 
-# `modular build <packages...>`
+# `modular build [packages...]`
 
 Search workspaces based on their `name` field in the `package.json` and build
-them according to their respective `modular.type`.
+them according to their respective `modular.type`, in order of dependency (e.g.
+if a package `a` depends on a package `b`, `b` is built first).
 
 The output directory for built artifacts is `dist/`, which has a flat structure
 of modular package names. Each built app/view/package is added to the `dist/` as
@@ -20,6 +21,135 @@ this-is-param-case) in `dist/`
 
 ## Options:
 
-`--private`: Allows the building of private packages
+`--private`: Allows the building of private packages.
 
-`--preserve-modules`: Preserve module structure in generated modules
+`--preserve-modules`: Preserve module structure in generated modules.
+
+`--changed`: Build only packages whose workspaces contain files that have
+changed. Files that have changed are calculated comparing the current state of
+the repository with the branch specified by `compareBranch` or, if
+`compareBranch` is not set, with the default git branch.
+
+`--compareBranch`: Specify the comparison branch used to determine which files
+have changed when using the `changed` option. If this option is used without
+`changed`, the command will fail.
+
+`--descendants`: Build the packages specified by the `[packages...]` argument
+and/or the `--changed` option and additionally build all their descendants (i.e.
+the packages they directly or indirectly depend on) in dependency order.
+
+`--ancestors`: Build the packages specified by the `[packages...]` argument
+and/or the `--changed` option and additionally build all their ancestors (i.e.
+the packages that have a direct or indirect dependency on them) in dependency
+order.
+
+## Dependency selection and build order examples
+
+We'll be using this package manifests in our Modular monorepo for the following
+examples:
+
+```js
+{
+  "name": "a",
+  "dependencies": {
+    "b": "*",
+    "c": "*",
+    "react": ">16.8.0",
+    // ...
+  }
+}
+
+{
+  "name": "b",
+  "dependencies": {
+    "c": "*",
+    "react": ">16.8.0",
+    // ...
+  }
+}
+
+{
+  "name": "c",
+  "dependencies": {
+    "d": "*",
+    // ...
+  }
+}
+
+{
+  "name": "d",
+  "dependencies": {}
+}
+
+{
+  "name": "e",
+  "dependencies": {
+    "a": "*",
+    // ...
+  }
+}
+```
+
+Which internally are filtered into this set of `WorkspaceDependencyObject`s:
+
+```js
+{
+  a: { workspaceDependencies: ['b', 'c'] },
+  b: { workspaceDependencies: ['c'] },
+  c: { workspaceDependencies: ['d'] },
+  d: { workspaceDependencies: undefined },
+  e: { workspaceDependencies: ['a'] }
+}
+```
+
+### Example: local workflow with descendants
+
+Let's say we just pulled an update to our monorepo and we want to work on
+workspace `b`. To be able to work with the last modifications we pulled, we want
+to build `b` and all the other workspaces that `b` depends on (descendants),
+either directly or indirectly. We can tell Modular that we want to build `b` and
+its all descendants by using this command:
+
+`modular build b --descendants`
+
+Modular will first select all the descendants of `b` (according to the previous
+graph: `c` because it'a direct dependency and `d` because it's a dependency of
+`c`), then build them in the correct build order, where workspaces depended on
+are built before workspaces that depend on them, recursively. In this example:
+
+- `d` gets built first, because it has no dependencies
+- `c` can now get built, because it only depends on `d`, that got built in the
+  previous step.
+- `b` can now get built, because it only depends on `c`, that got built in the
+  previous step.
+
+### Example: incremental builds with ancestors
+
+Let's suppose we're building a PR of our monorepository on a CI pipeline, and we
+want to incrementally build the workspaces that have code changes compared to
+the base branch. Since those workspaces will generate new, different build
+artefacts, we can't just build them and call it a day; we also need to re-build
+all the workspaces that depend on the changed workspaces, and those who depend
+on them, and so on. In other words, we need a way to tell Modular to build the
+ancestors of the changed workspaces. This command:
+
+`modular build --changed --ancestors`
+
+will identify all the workspaces that have changed compared to the
+`--compareBranch` (which is the repository's base branch by default), then
+identify all the workspaces which directly or indirectly depend on them
+(ancestors) and build the resulting set of packages in the correct build order,
+where workspaces depended on are built before workspaces that depend on them,
+recursively. If we suppose that workspaces `b` and `c` have changed, Modular
+will:
+
+- Select all ancestors of `b` and `c`, which are `a` (because it depends on
+  both) and `e` (because it depends on `a`).
+- Build `c` first, because it doesn't depend on any package that has changed (it
+  only depends on `d`, which is not in the changed set).
+- Build `b`, because it only depends on `c`, that got built in the previous
+  step.
+- Build `a`, because it depends on `b` and `c`, that got built in the previous
+  steps.
+- Build `e`, because it only depends on `a`, that got built in the previous
+  step.