feat: add opt-in exports_directories_only mode to yarn_install and npm_install (defaults to False)

When `True`, `yarn_install` and `npm_install` export only top-level package directory artifacts from node_modules.

Turning this on will decrease the time it takes for Bazel to setup runfiles and sandboxing when
there are a large number of npm dependencies as inputs to an action.

This breaks compatabilty for label that reference files within npm packages such as `@npm//:node_modules/prettier/bin-prettier.js`.
To reference files within npm packages, you can use the `directory_file_path` rule and/or `DirectoryFilePathInfo` provider.
Note, some rules still need upgrading to support consuming `DirectoryFilePathInfo` where needed.

NB: This feature requires runfiles be enabled due to an issue in Bazel which we are still investigating.
    On Windows runfiles are off by default and must be enabled with the `--enable_runfiles` flag when
    using this feature.

NB: `ts_library` does not support directory artifact npm deps due to internal dependency on having all input sources files explicitly specified

NB: `karma_web_test` and `karma_web_test_suite` do not yet support directory artifact npm deps as they require `node_modules/requirejs/require.js` & `node_modules/karma-requirejs/lib/adapter.js` explicit source files in deps

For the `nodejs_binary` & `nodejs_test` `entry_point` attribute (which often needs to reference a file within
an npm package) you can set the entry_point to a dict with a single entry, where the key corresponds to the directory artifact
label and the value corresponds to the path within that directory to the entry point.

For example,

```
nodejs_binary(
    name = "prettier",
    data = ["@npm//prettier"],
    entry_point = "@npm//:node_modules/prettier/bin-prettier.js",
)
```

becomes,

```
nodejs_binary(
    name = "prettier",
    data = ["@npm//prettier"],
    entry_point = { "@npm//:node_modules/prettier": "bin-prettier.js" },
)
```

For other labels that are passed to `$(rootpath)`, `$(execpath)`, or `$(location)` you can simply break these apart into
the directory artifact label that gets passed to the expander & path part to follows it.

For example,

```
$(rootpath @npm//:node_modules/prettier/bin-prettier.js")
```

becomes,

```
$(rootpath @npm//:node_modules/prettier)/bin-prettier.js
```

Author: Greg Magolan

.bazelci/presubmit.yml

@@ -382,65 +382,74 @@ tasks:
     # Disabled due to https://github.com/bazelbuild/rules_nodejs/issues/1486
     #- "@fine_grained_deps_yarn//typescript/bin:tsc"
     build_flags:
-    - "--build_tag_filters=-e2e,-examples,-fix-windows,-manual"
+    - "--build_tag_filters=-e2e,-examples,-manual,-fix-windows,-no-bazelci-windows,-requires-runfiles"
     build_targets:
     - "//..."
     test_flags:
     # Firefox not supported on Windows with rules_webtesting (if run it exit with success)
     # See https://github.com/bazelbuild/rules_webtesting/blob/0.3.3/browsers/BUILD.bazel#L66.
-    - "--test_tag_filters=-e2e,-examples,-fix-windows,-no-bazelci-windows,-manual,-browser:firefox-local,-cypress"
+    - "--test_tag_filters=-e2e,-examples,-fix-windows,-no-bazelci-windows,-requires-runfiles,-manual,-browser:firefox-local,-cypress"
     test_targets:
     - "//..."
     # //internal/node/test:nodejs_toolchain_windows_amd64_test is a "manual" test that must be run
     # explicitly; it should pass when running on Windows with no --platform set.
     - "//internal/node/test:nodejs_toolchain_windows_amd64_test"
+  windows_runfiles_enabled:
+    name: windows_runfiles_enabled
+    platform: windows
+    test_flags:
+    - "--test_tag_filters=requires-runfiles"
+    - "--enable_runfiles"
+    test_targets:
+    - "//..."
   windows_e2e:
     name: windows_e2e
     platform: windows
     build_flags:
-    - "--build_tag_filters=e2e,-fix-windows"
+    - "--build_tag_filters=e2e,-fix-windows,-no-bazelci-windows,-requires-runfiles"
     build_targets:
     - "//..."
     # We control Bazel version in integration tests, so we don't need USE_BAZEL_VERSION for tests.
     skip_use_bazel_version_for_test: true
     test_flags:
-    - "--test_tag_filters=e2e,-fix-windows,-no-bazelci-windows"
+    - "--test_tag_filters=e2e,-fix-windows,-no-bazelci-windows,-requires-runfiles"
     - "--local_ram_resources=792"
     # test_args will be passed to the nested bazel process
     - "--test_arg=--local_ram_resources=13288"
     # Firefox not supported on Windows with rules_webtesting (if run it exit with success)
     # See https://github.com/bazelbuild/rules_webtesting/blob/0.3.3/browsers/BUILD.bazel#L66.
     # Chrome fails to launch on Windows inside bazel-in-bazel with: [17:12:04] E/launcher -
     # spawn D:\...\external\io_bazel_rules_webtesting\third_party\chromedriver\chromedriver.out\chromedriver_win32\chromedriver.exe ENOENT
-    - "--test_arg=--test_tag_filters=-fix-windows,-manual,-browser:chromium-local,-browser:firefox-local"
+    - "--test_arg=--test_tag_filters=-fix-windows,-no-bazelci-windows,-requires-runfiles,-manual,-browser:chromium-local,-browser:firefox-local"
     test_targets:
     - "//..."
   windows_examples:
     name: windows_examples
     platform: windows
     build_flags:
-    - "--build_tag_filters=examples,-fix-windows"
+    - "--build_tag_filters=examples,-fix-windows,-no-bazelci-windows,-requires-runfiles"
     build_targets:
     - "//..."
     # We control Bazel version in integration tests, so we don't need USE_BAZEL_VERSION for tests.
     skip_use_bazel_version_for_test: true
     test_flags:
-    - "--test_tag_filters=examples,-manual,-fix-windows,-no-bazelci-windows,-cypress"
+    - "--test_tag_filters=examples,-manual,-fix-windows,-no-bazelci-windows,-requires-runfiles,-cypress"
     - "--local_ram_resources=792"
     # test_args will be passed to the nested bazel process
     - "--test_arg=--local_ram_resources=13288"
     # Firefox not supported on Windows with rules_webtesting (if run it exit with success)
     # See https://github.com/bazelbuild/rules_webtesting/blob/0.3.3/browsers/BUILD.bazel#L66.
     # Chrome fails to launch on Windows inside bazel-in-bazel with: [17:12:04] E/launcher -
     # spawn D:\...\external\io_bazel_rules_webtesting\third_party\chromedriver\chromedriver.out\chromedriver_win32\chromedriver.exe ENOENT
-    - "--test_arg=--test_tag_filters=-no-bazelci-windows,-fix-windows,-manual,-browser:chromium-local,-browser:firefox-local,-cypress"
+    - "--test_arg=--test_tag_filters=-fix-windows,-no-bazelci-windows,-requires-runfiles,-manual,-browser:chromium-local,-browser:firefox-local,-cypress"
     test_targets:
     - "//..."
   windows_cross_compile_linux:
     name: windows_cross_compile_linux
     platform: windows
-    # Build on windows with the node --platform set to linux
     build_flags:
+    - "--build_tag_filters=-fix-windows,-no-bazelci-windows,-requires-runfiles"
+    # Build on windows with the node --platform set to linux
     - "--platforms=@build_bazel_rules_nodejs//toolchains/node:linux_amd64"
     build_targets:
     - "//internal/..."

.bazelrc

@@ -5,8 +5,8 @@ import %workspace%/common.bazelrc
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # To update these lines, just run `yarn bazel:update-deleted-packages`
 # (Note, we cannot use common --deleted_packages because the bazel version command doesn't support it)
-build --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/kotlin/node_modules/@bazel/rollup,examples/kotlin/node_modules/@bazel/terser,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
-query --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/kotlin/node_modules/@bazel/rollup,examples/kotlin/node_modules/@bazel/terser,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
+build --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/concatjs_devserver_directory_artifacts,e2e/concatjs_devserver_directory_artifacts/genrule,e2e/concatjs_devserver_directory_artifacts/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
+query --deleted_packages=examples/angular,examples/angular/e2e,examples/angular/src/app/home,examples/angular/src/app/hello-world,examples/angular/src/app,examples/angular/src/app/todos,examples/angular/src/app/todos/reducers,examples/angular/src/shared/material,examples/angular/src/lib/shorten,examples/angular/src,examples/angular/src/assets,examples/angular/tools,examples/angular_bazel_architect,examples/angular_bazel_architect/projects/frontend-lib,examples/angular_view_engine,examples/angular_view_engine/e2e,examples/angular_view_engine/src/app/home,examples/angular_view_engine/src/app/hello-world,examples/angular_view_engine/src/app,examples/angular_view_engine/src/app/todos,examples/angular_view_engine/src/app/todos/reducers,examples/angular_view_engine/src/shared/material,examples/angular_view_engine/src/lib/typography,examples/angular_view_engine/src/lib/shorten,examples/angular_view_engine/src,examples/angular_view_engine/src/assets,examples/app,examples/app/styles,examples/app/test,examples/closure,examples/create-react-app,examples/cypress,examples/esbuild,examples/esbuild/src,examples/from_source,examples/jest,examples/jest/ts/test,examples/jest/ts,examples/jest/ts/src,examples/kotlin,examples/nestjs,examples/nestjs/src,examples/parcel,examples/protobufjs,examples/protocol_buffers,examples/react_webpack,examples/user_managed_deps,examples/vendored_node,examples/vendored_node_and_yarn,examples/vue,examples/vue/src,examples/web_testing,examples/webapp,examples/worker,e2e/bazel_managed_deps,e2e/concatjs_devserver,e2e/concatjs_devserver/genrule,e2e/concatjs_devserver/subpackage,e2e/concatjs_devserver_directory_artifacts,e2e/concatjs_devserver_directory_artifacts/genrule,e2e/concatjs_devserver_directory_artifacts/subpackage,e2e/coverage,e2e/fine_grained_symlinks,e2e/jasmine,e2e/node_loader_no_preserve_symlinks,e2e/node_loader_preserve_symlinks,e2e/nodejs_image,e2e/nodejs_image/foolib,e2e/nodejs_repository,e2e/packages,e2e/symlinked_node_modules_npm,e2e/symlinked_node_modules_yarn,e2e/typescript,e2e/webapp
 
 # Mock versioning command to test the --stamp behavior
 build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"

docs/Built-ins.md

@@ -1771,7 +1771,7 @@ Defaults to `[]`
 
 The name it will be imported by. Should match the "name" field in the package.json file.
 
-If package_name == "$node_modules$" this indictates that this js_library target is one or more external npm
+If package_name is “$node_modules$” or “$node_modules_dir$” this indictates that this js_library target is one or more external npm
 packages in node_modules. This is a special case that used be covered by the internal only
 `external_npm_package` attribute. NB: '$' is an illegal character
 for npm packages names so this reserved name will not conflict with any valid package_name values

e2e/BUILD.bazel

@@ -122,6 +122,21 @@ e2e_integration_test(
     },
 )
 
+e2e_integration_test(
+    name = "e2e_concatjs_devserver_directory_artifacts",
+    bazel_commands = [
+        "info",
+        # enable runfiles explicitly so it is on for Windows since directory
+        # artifacts requires runfiles
+        "test ... --enable_runfiles",
+    ],
+    npm_packages = {
+        "//packages/concatjs:npm_package": "@bazel/concatjs",
+        "//packages/protractor:npm_package": "@bazel/protractor",
+        "//packages/typescript:npm_package": "@bazel/typescript",
+    },
+)
+
 [e2e_integration_test(
     name = "e2e_typescript_%s" % tsc_version.replace(".", "_"),
     npm_packages = {

e2e/concatjs_devserver_directory_artifacts/.bazelignore

@@ -0,0 +1 @@
+node_modules

e2e/concatjs_devserver_directory_artifacts/.bazelrc

@@ -0,0 +1 @@
+import %workspace%/../../common.bazelrc

e2e/concatjs_devserver_directory_artifacts/BUILD.bazel

@@ -0,0 +1,85 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("@build_bazel_rules_nodejs//internal/npm_install:npm_umd_bundle.bzl", "npm_umd_bundle")
+load("@npm//@bazel/concatjs:index.bzl", "concatjs_devserver")
+load("@npm//@bazel/protractor:index.bzl", "protractor_web_test_suite")
+load("@npm//@bazel/typescript:index.bzl", "ts_project")
+
+exports_files([
+    "red-body-style.css",
+    "tsconfig.json",
+    "protractor.on-prepare.js",
+])
+
+ts_project(
+    name = "compile",
+    srcs = [
+        "app.ts",
+        "app_e2e-spec.ts",
+    ],
+    declaration = True,
+    tsconfig = ":tsconfig.json",
+    validate = False,
+    deps = [
+        "@npm//@types",
+        "@npm//date-fns",
+        "@npm//protractor",
+        "@npm//rxjs",
+        "@npm//typeorm",
+    ],
+)
+
+js_library(
+    name = "lib",
+    named_module_srcs = [":compile"],
+    deps = [":compile"],
+)
+
+# typeorm is a special case for npm_umd_bundle as we need to excluded
+# the `react-native-sqlite-storage` dynamic require from the umd bundle
+npm_umd_bundle(
+    name = "typeorm_umd",
+    package_name = "typeorm",
+    entry_point = {"@npm//:node_modules/typeorm": "browser/index.js"},
+    excluded = ["react-native-sqlite-storage"],
+    package = "@npm//typeorm",
+)
+
+concatjs_devserver(
+    name = "devserver",
+    entry_module = "concatjs_devserver_directory_artifacts/app",
+    scripts = [
+        "@npm//date-fns:date-fns.umd.js",
+        "@npm//rxjs:rxjs.umd.js",
+        ":typeorm.umd.js",
+    ],
+    # We'll collect all the devmode JS sources from these TypeScript libraries
+    deps = [":lib"],
+)
+
+protractor_web_test_suite(
+    name = "devserver_test",
+    on_prepare = ":protractor.on-prepare.js",
+    protractor_entry_point = {"@npm//:node_modules/protractor": "bin/protractor"},
+    server = ":devserver",
+    deps = [":lib"],
+)
+
+# Just a dummy test so that we have a test target for //... on certain bazelci platforms with bazel_integration_test
+sh_test(
+    name = "dummy_test",
+    srcs = ["dummy_test.sh"],
+)

e2e/concatjs_devserver_directory_artifacts/WORKSPACE

@@ -0,0 +1,51 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+workspace(
+    name = "concatjs_devserver_directory_artifacts",
+    managed_directories = {"@npm": ["node_modules"]},
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "build_bazel_rules_nodejs",
+    sha256 = "4a5d654a4ccd4a4c24eca5d319d85a88a650edf119601550c95bf400c8cc897e",
+    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.5.1/rules_nodejs-3.5.1.tar.gz"],
+)
+
+load("@build_bazel_rules_nodejs//:index.bzl", "yarn_install")
+
+yarn_install(
+    name = "npm",
+    exports_directories_only = True,
+    package_json = "//:package.json",
+    yarn_lock = "//:yarn.lock",
+)
+
+load("@npm//@bazel/protractor:package.bzl", "npm_bazel_protractor_dependencies")
+
+npm_bazel_protractor_dependencies()
+
+# Setup the rules_webtesting toolchain
+load("@io_bazel_rules_webtesting//web:repositories.bzl", "web_test_repositories")
+
+web_test_repositories()
+
+load("@io_bazel_rules_webtesting//web/versioned:browsers-0.3.2.bzl", "browser_repositories")
+
+browser_repositories(
+    chromium = True,
+    firefox = True,
+)

e2e/concatjs_devserver_directory_artifacts/app.ts

@@ -0,0 +1,46 @@
+/// <amd-module name="concatjs_devserver_directory_artifacts/app"/>
+
+import {format} from 'date-fns';
+import {of} from 'rxjs';
+import {Column, Entity, PrimaryGeneratedColumn} from 'typeorm';
+
+function appendText(text: string, className: string) {
+  const el: HTMLDivElement = document.createElement('div');
+  el.innerText = text;
+  el.className = className;
+  document.body.appendChild(el);
+}
+
+function testDateFns() {
+  const date: string = format(new Date(2019, 4, 7), 'MMMM D, YYYY');
+  appendText(`Hello, TypeScript today is ${date}`, 'ts1');
+}
+
+
+@Entity()
+export class User {
+  @PrimaryGeneratedColumn()
+  id: number;
+  @Column()
+  firstName: string;
+  @Column()
+  lastName: string;
+  @Column()
+  isActive: boolean;
+}
+
+function testBrowserEntryPoint() {
+  // this script will throw if the browser entryPoint in typeorm wasn't resolved correctly
+  const user = new User();
+  user.firstName = 'foo';
+  appendText(`firstname: ${user.firstName}`, 'entrypoint-browser');
+}
+
+function testModuleEntryPoint() {
+  // rxjs uses the module entrypoint to resolve to some import/export
+  of('rxjs works with modules!').subscribe(value => appendText(String(value), 'entrypoint-module'))
+}
+
+testDateFns();
+testBrowserEntryPoint();
+testModuleEntryPoint();