Javascript backend - run node package install once per workspace

docs/notes/2.23.x.md

@@ -121,6 +121,13 @@ pants to invoke `corepack`s default versions of the package managers instead.
 (npm, pnpm, yarn) without providing a corresponding `[nodejs].package_managers` version setting. The version is then
 entirely handled by `corepack`. Previously this mode caused pants to fail.
 
+The internal installation mechanism for node_modules has changed. 
+Previously, Pants installed each package separately in sandboxes and merged the results, creating a node_modules for all dependent packages in the workspace. 
+Now, this is delegated to the package managers, using each package manager's support for workspaces.
+
+This fixes an issue with integrity file collisions when newer versions of package managers (e.g. the [hidden lockfiles](https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#hidden-lockfiles) introduced in npm v7).
+
+
 #### Shell
 
 The `tailor` goal now has independent options for tailoring `shell_sources` and `shunit2_tests` targets. The option was split from `tailor` into [`tailor_sources`](https://www.pantsbuild.org/2.22/reference/subsystems/shell-setup#tailor_sources) and [`tailor_shunit2_tests`](https://www.pantsbuild.org/2.22/reference/subsystems/shell-setup#tailor_shunit2_tests).

src/python/pants/backend/javascript/goals/test_integration_test.py

@@ -297,7 +297,7 @@ def test_jest_test_with_coverage_reporting(
                             coverage_args=['--coverage', '--coverage-directory=.coverage/'],
                             coverage_output_files=['.coverage/clover.xml'],
                         )
-                    ]
+                    ],
                 )
                 """
             ),

src/python/pants/backend/javascript/install_node_package.py

@@ -2,7 +2,6 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 from __future__ import annotations
 
-import itertools
 import os.path
 from dataclasses import dataclass
 from typing import Iterable
@@ -25,16 +24,10 @@
 from pants.core.target_types import FileSourceField, ResourceSourceField
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.engine.internals.native_engine import AddPrefix, Digest, MergeDigests
-from pants.engine.internals.selectors import Get, MultiGet
+from pants.engine.internals.selectors import Get
 from pants.engine.process import ProcessResult
 from pants.engine.rules import Rule, collect_rules, rule
-from pants.engine.target import (
-    SourcesField,
-    Target,
-    TransitiveTargets,
-    TransitiveTargetsRequest,
-    targets_with_sources_types,
-)
+from pants.engine.target import SourcesField, Target, TransitiveTargets, TransitiveTargetsRequest
 from pants.engine.unions import UnionMembership, UnionRule
 
 
@@ -87,43 +80,35 @@ async def install_node_packages_for_address(
     target = project_env.ensure_target()
     transitive_tgts = await Get(TransitiveTargets, TransitiveTargetsRequest([target.address]))
 
-    pkg_tgts = targets_with_sources_types(
-        [PackageJsonSourceField], transitive_tgts.dependencies, union_membership
-    )
-    assert target not in pkg_tgts
-    installations = await MultiGet(
-        Get(InstalledNodePackageWithSource, InstalledNodePackageRequest(pkg_tgt.address))
-        for pkg_tgt in pkg_tgts
-    )
-
     source_files = await _get_relevant_source_files(
         (tgt[SourcesField] for tgt in transitive_tgts.closure if tgt.has_field(SourcesField)),
         with_js=False,
     )
-    install_input_digest = await Get(
-        Digest,
-        MergeDigests(
-            itertools.chain(
-                (installation.digest for installation in installations),
-                (source_files.snapshot.digest,),
-            )
-        ),
-    )
+    package_digest = source_files.snapshot.digest
 
     install_result = await Get(
         ProcessResult,
         NodeJsProjectEnvironmentProcess(
             project_env,
             project_env.project.immutable_install_args,
             description=f"Installing {target[NodePackageNameField].value}@{target[NodePackageVersionField].value}.",
-            input_digest=install_input_digest,
+            input_digest=package_digest,
             output_directories=tuple(project_env.node_modules_directories),
         ),
     )
     node_modules = await Get(Digest, AddPrefix(install_result.output_digest, project_env.root_dir))
+
     return InstalledNodePackage(
         project_env,
-        digest=await Get(Digest, MergeDigests([install_input_digest, node_modules])),
+        digest=await Get(
+            Digest,
+            MergeDigests(
+                [
+                    package_digest,
+                    node_modules,
+                ]
+            ),
+        ),
     )
 
 

src/python/pants/backend/javascript/nodejs_project.py

@@ -236,6 +236,8 @@ async def find_node_js_projects(
     nodejs: NodeJS,
     resolve_names: UserChosenNodeJSResolveAliases,
 ) -> AllNodeJSProjects:
+    # Note: If pnpm_workspace.yaml is present for an npm-managed project, it will override the package.json["workspaces"] setting, which is not intuitive
+    # Probably pnpm_workspace.yaml should only be used for pnpm projects.
     project_paths = (
         ProjectPaths(pkg.root_dir, ["", *pkg.workspaces])
         if pkg not in pnpm_workspaces

src/python/pants/backend/javascript/nodejs_project_environment.py

@@ -125,12 +125,6 @@ async def setup_nodejs_project_environment_process(req: NodeJsProjectEnvironment
         args = tuple(req.args)
     output_files = req.output_files
     output_directories = req.output_directories
-    per_package_caches = FrozenDict(
-        {
-            key: os.path.join(req.env.package_dir(), value)
-            for key, value in req.per_package_caches.items()
-        }
-    )
     return await Get(
         Process,
         NodeJSToolProcess,
@@ -144,7 +138,7 @@ async def setup_nodejs_project_environment_process(req: NodeJsProjectEnvironment
             working_directory=req.env.root_dir,
             output_files=output_files,
             output_directories=output_directories,
-            append_only_caches=FrozenDict(**per_package_caches, **req.env.project.extra_caches()),
+            append_only_caches=FrozenDict(**req.env.project.extra_caches()),
             timeout_seconds=req.timeout_seconds,
             project_digest=project_digest,
             extra_env=FrozenDict(**req.extra_env, **req.env.project.extra_env()),