From e5fecbbdfff1169ce9bf56254b508d0775ae5a48 Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Wed, 27 Jul 2022 15:22:44 -0400 Subject: [PATCH 1/2] implement the yarn pnp module resolution algorithm --- internal/js_ast/js_ast.go | 3 + internal/js_parser/js_parser.go | 73 +++ internal/js_parser/json_parser.go | 31 ++ internal/logger/logger.go | 162 ++++++ internal/resolver/resolver.go | 31 ++ internal/resolver/testExpectations.json | 311 ++++++++++++ internal/resolver/yarnpnp.go | 631 ++++++++++++++++++++++++ internal/resolver/yarnpnp_test.go | 90 ++++ scripts/js-api-tests.js | 254 ++++++++++ 9 files changed, 1586 insertions(+) create mode 100644 internal/resolver/testExpectations.json create mode 100644 internal/resolver/yarnpnp.go create mode 100644 internal/resolver/yarnpnp_test.go diff --git a/internal/js_ast/js_ast.go b/internal/js_ast/js_ast.go index 2dd74593e1f..11b5c604d6d 100644 --- a/internal/js_ast/js_ast.go +++ b/internal/js_ast/js_ast.go @@ -1771,6 +1771,9 @@ type AST struct { ModuleScope *Scope CharFreq *CharFreq + // This is internal-only data used for the implementation of Yarn PnP + ManifestForYarnPnP Expr + Hashbang string Directive string URLForCSS string diff --git a/internal/js_parser/js_parser.go b/internal/js_parser/js_parser.go index 977901c05b0..ecc82576643 100644 --- a/internal/js_parser/js_parser.go +++ b/internal/js_parser/js_parser.go @@ -149,6 +149,10 @@ type parser struct { loopBody js_ast.S moduleScope *js_ast.Scope + // This is internal-only data used for the implementation of Yarn PnP + manifestForYarnPnP js_ast.Expr + stringLocalsForYarnPnP map[js_ast.Ref]stringLocalForYarnPnP + // This helps recognize the "await import()" pattern. When this is present, // warnings about non-string import paths will be omitted inside try blocks. awaitTarget js_ast.E @@ -341,6 +345,11 @@ type parser struct { isControlFlowDead bool } +type stringLocalForYarnPnP struct { + value []uint16 + loc logger.Loc +} + type injectedSymbolSource struct { source logger.Source loc logger.Loc @@ -414,6 +423,17 @@ type optionsThatSupportStructuralEquality struct { mangleQuoted bool unusedImportFlagsTS config.UnusedImportFlagsTS useDefineForClassFields config.MaybeBool + + // This is an internal-only option used for the implementation of Yarn PnP + decodeHydrateRuntimeStateYarnPnP bool +} + +func OptionsForYarnPnP() Options { + return Options{ + optionsThatSupportStructuralEquality: optionsThatSupportStructuralEquality{ + decodeHydrateRuntimeStateYarnPnP: true, + }, + } } func OptionsFromConfig(options *config.Options) Options { @@ -9463,6 +9483,18 @@ func (p *parser) visitAndAppendStmt(stmts []js_ast.Stmt, stmt js_ast.Stmt) []js_ } } } + + // Yarn's PnP data may be stored in a variable: https://github.com/yarnpkg/berry/pull/4320 + if p.options.decodeHydrateRuntimeStateYarnPnP { + if str, ok := d.ValueOrNil.Data.(*js_ast.EString); ok { + if id, ok := d.Binding.Data.(*js_ast.BIdentifier); ok { + if p.stringLocalsForYarnPnP == nil { + p.stringLocalsForYarnPnP = make(map[js_ast.Ref]stringLocalForYarnPnP) + } + p.stringLocalsForYarnPnP[id.Ref] = stringLocalForYarnPnP{value: str.Value, loc: d.ValueOrNil.Loc} + } + } + } } // Attempt to continue the const local prefix @@ -14039,6 +14071,46 @@ func (p *parser) visitExprInOut(expr js_ast.Expr, in exprIn) (js_ast.Expr, exprO e.Args[i] = arg } + // Our hack for reading Yarn PnP files is implemented here: + if p.options.decodeHydrateRuntimeStateYarnPnP { + if id, ok := e.Target.Data.(*js_ast.EIdentifier); ok && p.symbols[id.Ref.InnerIndex].OriginalName == "hydrateRuntimeState" && len(e.Args) >= 1 { + switch arg := e.Args[0].Data.(type) { + case *js_ast.EObject: + // "hydrateRuntimeState()" + if arg := e.Args[0]; isValidJSON(arg) { + p.manifestForYarnPnP = arg + } + + case *js_ast.ECall: + // "hydrateRuntimeState(JSON.parse())" + if len(arg.Args) == 1 { + if dot, ok := arg.Target.Data.(*js_ast.EDot); ok && dot.Name == "parse" { + if id, ok := dot.Target.Data.(*js_ast.EIdentifier); ok { + if symbol := &p.symbols[id.Ref.InnerIndex]; symbol.Kind == js_ast.SymbolUnbound && symbol.OriginalName == "JSON" { + arg := arg.Args[0] + switch a := arg.Data.(type) { + case *js_ast.EString: + // "hydrateRuntimeState(JSON.parse())" + source := logger.Source{KeyPath: p.source.KeyPath, Contents: helpers.UTF16ToString(a.Value)} + log := logger.NewStringInJSLog(p.log, &p.tracker, arg.Loc, source.Contents) + p.manifestForYarnPnP, _ = ParseJSON(log, source, JSONOptions{}) + + case *js_ast.EIdentifier: + // "hydrateRuntimeState(JSON.parse())" + if data, ok := p.stringLocalsForYarnPnP[a.Ref]; ok { + source := logger.Source{KeyPath: p.source.KeyPath, Contents: helpers.UTF16ToString(data.value)} + log := logger.NewStringInJSLog(p.log, &p.tracker, data.loc, source.Contents) + p.manifestForYarnPnP, _ = ParseJSON(log, source, JSONOptions{}) + } + } + } + } + } + } + } + } + } + // Stop now if this call must be removed if out.methodCallMustBeReplacedWithUndefined { p.isControlFlowDead = oldIsControlFlowDead @@ -16486,6 +16558,7 @@ func (p *parser) toAST(before, parts, after []js_ast.Part, hashbang string, dire ApproximateLineCount: int32(p.lexer.ApproximateNewlineCount) + 1, MangledProps: p.mangledProps, ReservedProps: p.reservedProps, + ManifestForYarnPnP: p.manifestForYarnPnP, // CommonJS features UsesExportsRef: usesExportsRef, diff --git a/internal/js_parser/json_parser.go b/internal/js_parser/json_parser.go index e17b9003527..8a77adc29a0 100644 --- a/internal/js_parser/json_parser.go +++ b/internal/js_parser/json_parser.go @@ -187,3 +187,34 @@ func ParseJSON(log logger.Log, source logger.Source, options JSONOptions) (resul p.lexer.Expect(js_lexer.TEndOfFile) return } + +func isValidJSON(value js_ast.Expr) bool { + switch e := value.Data.(type) { + case *js_ast.ENull, *js_ast.EBoolean, *js_ast.EString, *js_ast.ENumber: + return true + + case *js_ast.EArray: + for _, item := range e.Items { + if !isValidJSON(item) { + return false + } + } + return true + + case *js_ast.EObject: + for _, property := range e.Properties { + if property.Kind != js_ast.PropertyNormal || property.Flags&(js_ast.PropertyIsComputed|js_ast.PropertyIsMethod) != 0 { + return false + } + if _, ok := property.Key.Data.(*js_ast.EString); !ok { + return false + } + if !isValidJSON(property.ValueOrNil) { + return false + } + } + return true + } + + return false +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 2b2d8edf5e2..323b86067e1 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -1686,3 +1686,165 @@ func allowOverride(overrides map[MsgID]LogLevel, id MsgID, kind MsgKind) (MsgKin } return kind, true } + +// For Yarn PnP we sometimes parse JSON embedded in a JS string. This is a shim +// that remaps log message locations inside the embedded string literal into +// log messages in the actual JS file, which makes them easier to understand. +func NewStringInJSLog(log Log, outerTracker *LineColumnTracker, outerStringLiteralLoc Loc, innerContents string) Log { + type entry struct { + line int32 + column int32 + loc Loc + } + + var table []entry + oldAddMsg := log.AddMsg + + generateTable := func() { + i := 0 + n := len(innerContents) + line := int32(1) + column := int32(0) + loc := Loc{Start: outerStringLiteralLoc.Start + 1} + outerContents := outerTracker.contents + + for i < n { + // Ignore line continuations. A line continuation is not an escaped newline. + for { + if c, _ := utf8.DecodeRuneInString(outerContents[loc.Start:]); c != '\\' { + break + } + c, width := utf8.DecodeRuneInString(outerContents[loc.Start+1:]) + switch c { + case '\n', '\r', '\u2028', '\u2029': + loc.Start += 1 + int32(width) + if c == '\r' && outerContents[loc.Start] == '\n' { + // Make sure Windows CRLF counts as a single newline + loc.Start++ + } + continue + } + break + } + + c, width := utf8.DecodeRuneInString(innerContents[i:]) + + // Compress the table using run-length encoding + table = append(table, entry{line: line, column: column, loc: loc}) + if len(table) > 1 { + if last := table[len(table)-2]; line == last.line && loc.Start-column == last.loc.Start-last.column { + table = table[:len(table)-1] + } + } + + // Advance the inner line/column + switch c { + case '\n', '\r', '\u2028', '\u2029': + line++ + column = 0 + + // Handle newlines on Windows + if c == '\r' && i+1 < n && innerContents[i+1] == '\n' { + i++ + } + + default: + column += int32(width) + } + i += width + + // Advance the outer loc, assuming the string syntax is already valid + c, width = utf8.DecodeRuneInString(outerContents[loc.Start:]) + if c == '\r' && outerContents[loc.Start] == '\n' { + // Handle newlines on Windows in template literal strings + loc.Start += 2 + } else if c != '\\' { + loc.Start += int32(width) + } else { + // Handle an escape sequence + c, width = utf8.DecodeRuneInString(outerContents[loc.Start+1:]) + switch c { + case 'x': + // 2-digit hexadecimal + loc.Start += 1 + 2 + + case 'u': + loc.Start++ + if outerContents[loc.Start] == '{' { + // Variable-length + for outerContents[loc.Start] != '}' { + loc.Start++ + } + loc.Start++ + } else { + // Fixed-length + loc.Start += 4 + } + + case '\n', '\r', '\u2028', '\u2029': + // This will be handled by the next iteration + break + + default: + loc.Start += int32(width) + } + } + } + } + + remapLineAndColumnToLoc := func(line int32, column int32) Loc { + count := len(table) + index := 0 + + // Binary search to find the previous entry + for count > 0 { + step := count / 2 + i := index + step + if i+1 < len(table) { + if entry := table[i+1]; entry.line < line || (entry.line == line && entry.column < column) { + index = i + 1 + count -= step + 1 + continue + } + } + count = step + } + + entry := table[index] + entry.loc.Start += column - entry.column // Undo run-length compression + return entry.loc + } + + remapData := func(data MsgData) MsgData { + if data.Location == nil { + return data + } + + // Generate and cache a lookup table to accelerate remappings + if table == nil { + generateTable() + } + + // Generate a range in the outer source using the line/column/length in the inner source + r := Range{Loc: remapLineAndColumnToLoc(int32(data.Location.Line), int32(data.Location.Column))} + if data.Location.Length != 0 { + r.Len = remapLineAndColumnToLoc(int32(data.Location.Line), int32(data.Location.Column+data.Location.Length)).Start - r.Loc.Start + } + + // Use that range to look up the line in the outer source + location := outerTracker.MsgData(r, data.Text).Location + location.Suggestion = data.Location.Suggestion + data.Location = location + return data + } + + log.AddMsg = func(msg Msg) { + msg.Data = remapData(msg.Data) + for i, note := range msg.Notes { + msg.Notes[i] = remapData(note) + } + oldAddMsg(msg) + } + + return log +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 181beab0bb6..3feead3d9a9 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -691,6 +691,18 @@ func (r resolverQuery) finalizeResolve(result *ResolveResult) { } func (r resolverQuery) resolveWithoutSymlinks(sourceDir string, sourceDirInfo *dirInfo, importPath string) *ResolveResult { + // Find the parent directory with the Yarn PnP data + for info := sourceDirInfo; info != nil; info = info.parent { + if info.pnpData != nil { + if result, ok := r.pnpResolve(importPath, sourceDirInfo.absPath, info.pnpData); ok { + importPath = result // Continue with the module resolution algorithm from node.js + } else { + return nil // This is a module resolution error + } + break + } + } + // This implements the module resolution algorithm from node.js, which is // described here: https://nodejs.org/api/modules.html#modules_all_together var result ResolveResult @@ -848,6 +860,7 @@ type dirInfo struct { // All relevant information about this directory absPath string entries fs.DirEntries + pnpData *pnpData packageJSON *packageJSON // Is there a "package.json" file in this directory? enclosingPackageJSON *packageJSON // Is there a "package.json" file in this directory or a parent directory? enclosingTSConfigJSON *TSConfigJSON // Is there a "tsconfig.json" file in this directory or a parent directory? @@ -1176,6 +1189,24 @@ func (r resolverQuery) dirInfoUncached(path string) *dirInfo { } } + // Record if this directory has a Yarn PnP data file + if pnp, _ := entries.Get(".pnp.data.json"); pnp != nil && pnp.Kind(r.fs) == fs.FileEntry { + absPath := r.fs.Join(path, ".pnp.data.json") + if json := r.extractYarnPnPDataFromJSON(absPath, &r.caches.JSONCache); json.Data != nil { + info.pnpData = compileYarnPnPData(absPath, path, json) + } + } else if pnp, _ := entries.Get(".pnp.cjs"); pnp != nil && pnp.Kind(r.fs) == fs.FileEntry { + absPath := r.fs.Join(path, ".pnp.cjs") + if json := r.tryToExtractYarnPnPDataFromJS(absPath, &r.caches.JSONCache); json.Data != nil { + info.pnpData = compileYarnPnPData(absPath, path, json) + } + } else if pnp, _ := entries.Get(".pnp.js"); pnp != nil && pnp.Kind(r.fs) == fs.FileEntry { + absPath := r.fs.Join(path, ".pnp.js") + if json := r.tryToExtractYarnPnPDataFromJS(absPath, &r.caches.JSONCache); json.Data != nil { + info.pnpData = compileYarnPnPData(absPath, path, json) + } + } + return info } diff --git a/internal/resolver/testExpectations.json b/internal/resolver/testExpectations.json new file mode 100644 index 00000000000..7a44b4339f1 --- /dev/null +++ b/internal/resolver/testExpectations.json @@ -0,0 +1,311 @@ +[{ + "manifest": { + "__info": [], + "dependencyTreeRoots": [{ + "name": "root", + "reference": "workspace:." + }], + "ignorePatternData": null, + "enableTopLevelFallback": false, + "fallbackPool": [], + "fallbackExclusionList": [], + "packageRegistryData": [ + [null, [ + [null, { + "packageLocation": "./", + "packageDependencies": [], + "linkType": "SOFT" + }] + ]], + ["root", [ + ["workspace:.", { + "packageLocation": "./", + "packageDependencies": [["test", "npm:1.0.0"]], + "linkType": "SOFT" + }] + ]], + ["workspace-alias-dependency", [ + ["workspace:workspace-alias-dependency", { + "packageLocation": "./workspace-alias-dependency/", + "packageDependencies": [["alias", ["test", "npm:1.0.0"]]], + "linkType": "SOFT" + }] + ]], + ["workspace-self-dependency", [ + ["workspace:workspace-self-dependency", { + "packageLocation": "./workspace-self-dependency/", + "packageDependencies": [["workspace-self-dependency", "workspace:workspace-self-dependency"]], + "linkType": "SOFT" + }] + ]], + ["workspace-unfulfilled-peer-dependency", [ + ["workspace:workspace-unfulfilled-peer-dependency", { + "packageLocation": "./workspace-unfulfilled-peer-dependency/", + "packageDependencies": [["test", null]], + "linkType": "SOFT" + }] + ]], + ["longer", [ + ["workspace:longer", { + "packageLocation": "./longer/", + "packageDependencies": [["test", "npm:2.0.0"]], + "linkType": "SOFT" + }] + ]], + ["long", [ + ["workspace:long", { + "packageLocation": "./long/", + "packageDependencies": [["test", "npm:1.0.0"]], + "linkType": "SOFT" + }] + ]], + ["longerer", [ + ["workspace:longerer", { + "packageLocation": "./longerer/", + "packageDependencies": [["test", "npm:3.0.0"]], + "linkType": "SOFT" + }] + ]], + ["test", [ + ["npm:1.0.0", { + "packageLocation": "./test-1.0.0/", + "packageDependencies": [], + "linkType": "HARD" + }], + ["npm:2.0.0", { + "packageLocation": "./test-2.0.0/", + "packageDependencies": [], + "linkType": "HARD" + }], + ["npm:3.0.0", { + "packageLocation": "./test-3.0.0/", + "packageDependencies": [], + "linkType": "HARD" + }] + ]] + ] + }, + "tests": [{ + "it": "should allow a package to import one of its dependencies", + "imported": "test", + "importer": "/path/to/project/", + "expected": "/path/to/project/test-1.0.0/" + }, { + "it": "should allow a package to import itself, if specified in its own dependencies", + "imported": "workspace-self-dependency", + "importer": "/path/to/project/workspace-self-dependency/", + "expected": "/path/to/project/workspace-self-dependency/" + }, { + "it": "should allow a package to import an aliased dependency", + "imported": "alias", + "importer": "/path/to/project/workspace-alias-dependency/", + "expected": "/path/to/project/test-1.0.0/" + }, { + "it": "shouldn't allow a package to import something that isn't one of its dependencies", + "imported": "missing-dependency", + "importer": "/path/to/project/", + "expected": "error!" + }, { + "it": "shouldn't accidentally discard the trailing slash from the package locations", + "imported": "test", + "importer": "/path/to/project/long/", + "expected": "/path/to/project/test-1.0.0/" + }, { + "it": "should throw an exception when trying to access an unfulfilled peer dependency", + "imported": "test", + "importer": "/path/to/project/workspace-unfulfilled-peer-dependency/", + "expected": "error!" + }] +}, { + "manifest": { + "__info": [], + "dependencyTreeRoots": [{ + "name": "root", + "reference": "workspace:." + }], + "ignorePatternData": null, + "enableTopLevelFallback": true, + "fallbackPool": [ + ["test-2", "npm:1.0.0"], + ["alias", ["test-1", "npm:1.0.0"]] + ], + "fallbackExclusionList": [[ + "workspace-no-fallbacks", + ["workspace:workspace-no-fallbacks"] + ]], + "packageRegistryData": [ + [null, [ + [null, { + "packageLocation": "./", + "packageDependencies": [["test-1", "npm:1.0.0"]], + "linkType": "SOFT" + }] + ]], + ["root", [ + ["workspace:.", { + "packageLocation": "./", + "packageDependencies": [["test-1", "npm:1.0.0"]], + "linkType": "SOFT" + }] + ]], + ["workspace-no-fallbacks", [ + ["workspace:workspace-no-fallbacks", { + "packageLocation": "./workspace-no-fallbacks/", + "packageDependencies": [], + "linkType": "SOFT" + }] + ]], + ["workspace-with-fallbacks", [ + ["workspace:workspace-with-fallbacks", { + "packageLocation": "./workspace-with-fallbacks/", + "packageDependencies": [], + "linkType": "SOFT" + }] + ]], + ["workspace-unfulfilled-peer-dependency", [ + ["workspace:workspace-unfulfilled-peer-dependency", { + "packageLocation": "./workspace-unfulfilled-peer-dependency/", + "packageDependencies": [ + ["test-1", null], + ["test-2", null] + ], + "linkType": "SOFT" + }] + ]], + ["test-1", [ + ["npm:1.0.0", { + "packageLocation": "./test-1/", + "packageDependencies": [], + "linkType": "HARD" + }] + ]], + ["test-2", [ + ["npm:1.0.0", { + "packageLocation": "./test-2/", + "packageDependencies": [], + "linkType": "HARD" + }] + ]] + ] + }, + "tests": [{ + "it": "should allow resolution coming from the fallback pool if enableTopLevelFallback is set to true", + "imported": "test-1", + "importer": "/path/to/project/", + "expected": "/path/to/project/test-1/" + }, { + "it": "should allow the fallback pool to contain aliases", + "imported": "alias", + "importer": "/path/to/project/", + "expected": "/path/to/project/test-1/" + }, { + "it": "shouldn't use the fallback pool when the importer package is listed in fallbackExclusionList", + "imported": "test-1", + "importer": "/path/to/project/workspace-no-fallbacks/", + "expected": "error!" + }, { + "it": "should implicitly use the top-level package dependencies as part of the fallback pool", + "imported": "test-2", + "importer": "/path/to/project/workspace-with-fallbacks/", + "expected": "/path/to/project/test-2/" + }, { + "it": "should throw an error if a resolution isn't in in the package dependencies, nor inside the fallback pool", + "imported": "test-3", + "importer": "/path/to/project/workspace-with-fallbacks/", + "expected": "error!" + }, { + "it": "should use the top-level fallback if a dependency is missing because of an unfulfilled peer dependency", + "imported": "test-1", + "importer": "/path/to/project/workspace-unfulfilled-peer-dependency/", + "expected": "/path/to/project/test-1/" + }, { + "it": "should use the fallback pool if a dependency is missing because of an unfulfilled peer dependency", + "imported": "test-2", + "importer": "/path/to/project/workspace-unfulfilled-peer-dependency/", + "expected": "/path/to/project/test-2/" + }] +}, { + "manifest": { + "__info": [], + "dependencyTreeRoots": [{ + "name": "root", + "reference": "workspace:." + }], + "ignorePatternData": null, + "enableTopLevelFallback": false, + "fallbackPool": [ + ["test", "npm:1.0.0"] + ], + "fallbackExclusionList": [], + "packageRegistryData": [ + [null, [ + [null, { + "packageLocation": "./", + "packageDependencies": [], + "linkType": "SOFT" + }] + ]], + ["root", [ + ["workspace:.", { + "packageLocation": "./", + "packageDependencies": [], + "linkType": "SOFT" + }] + ]], + ["test", [ + ["npm:1.0.0", { + "packageLocation": "./test-1/", + "packageDependencies": [], + "linkType": "HARD" + }] + ]] + ] + }, + "tests": [{ + "it": "should ignore the fallback pool if enableTopLevelFallback is set to false", + "imported": "test", + "importer": "/path/to/project/", + "expected": "error!" + }] +}, { + "manifest": { + "__info": [], + "dependencyTreeRoots": [{ + "name": "root", + "reference": "workspace:." + }], + "ignorePatternData": "^not-a-workspace(/|$)", + "enableTopLevelFallback": false, + "fallbackPool": [], + "fallbackExclusionList": [], + "packageRegistryData": [ + [null, [ + [null, { + "packageLocation": "./", + "packageDependencies": [], + "linkType": "SOFT" + }] + ]], + ["root", [ + ["workspace:.", { + "packageLocation": "./", + "packageDependencies": [["test", "npm:1.0.0"]], + "linkType": "SOFT" + }] + ]], + ["test", [ + ["npm:1.0.0", { + "packageLocation": "./test/", + "packageDependencies": [], + "linkType": "HARD" + }] + ]] + ] + }, + "tests": [{ + "it": "shouldn't go through PnP when trying to resolve dependencies from packages covered by ignorePatternData", + "imported": "test", + "importer": "/path/to/project/not-a-workspace/", + "expected": "error!" + }] +}] diff --git a/internal/resolver/yarnpnp.go b/internal/resolver/yarnpnp.go new file mode 100644 index 00000000000..53b77419d03 --- /dev/null +++ b/internal/resolver/yarnpnp.go @@ -0,0 +1,631 @@ +package resolver + +import ( + "fmt" + "regexp" + "strings" + + "github.com/evanw/esbuild/internal/cache" + "github.com/evanw/esbuild/internal/helpers" + "github.com/evanw/esbuild/internal/js_ast" + "github.com/evanw/esbuild/internal/js_parser" + "github.com/evanw/esbuild/internal/logger" +) + +// This file implements the Yarn PnP specification: https://yarnpkg.com/advanced/pnp-spec/ + +type pnpData struct { + // A list of package locators that are roots of the dependency tree. There + // will typically be one entry for each workspace in the project (always at + // least one, as the top-level package is a workspace by itself). + dependencyTreeRoots map[string]string + + // Keys are the package idents, values are sets of references. Combining the + // ident with each individual reference yields the set of affected locators. + fallbackExclusionList map[string]map[string]bool + + // A map of locators that all packages are allowed to access, regardless + // whether they list them in their dependencies or not. + fallbackPool map[string]pnpIdentAndReference + + // A nullable regexp. If set, all project-relative importer paths should be + // matched against it. If the match succeeds, the resolution should follow + // the classic Node.js resolution algorithm rather than the Plug'n'Play one. + // Note that unlike other paths in the manifest, the one checked against this + // regexp won't begin by `./`. + ignorePatternData *regexp.Regexp + + // This is the main part of the PnP data file. This table contains the list + // of all packages, first keyed by package ident then by package reference. + // One entry will have `null` in both fields and represents the absolute + // top-level package. + packageRegistryData map[string]map[string]pnpPackage + + packageLocatorsByLocations map[string]pnpPackageLocatorByLocation + + // If true, should a dependency resolution fail for an importer that isn't + // explicitly listed in `fallbackExclusionList`, the runtime must first check + // whether the resolution would succeed for any of the packages in + // `fallbackPool`; if it would, transparently return this resolution. Note + // that all dependencies from the top-level package are implicitly part of + // the fallback pool, even if not listed here. + enableTopLevelFallback bool + + absPath string + absDirPath string +} + +// This is called both a "locator" and a "dependency target" in the specification. +// When it's used as a dependency target, it can only be in one of three states: +// +// 1. A reference, to link with the dependency name +// In this case ident is "". +// +// 2. An aliased package +// In this case neither ident nor reference are "". +// +// 3. A missing peer dependency +// In this case ident and reference are "". +type pnpIdentAndReference struct { + ident string // Empty if null + reference string // Empty if null +} + +type pnpPackage struct { + packageDependencies map[string]pnpIdentAndReference + packageLocation string + discardFromLookup bool +} + +type pnpPackageLocatorByLocation struct { + locator pnpIdentAndReference + discardFromLookup bool +} + +// Note: If this returns successfully then the node module resolution algorithm +// (i.e. NM_RESOLVE in the Yarn PnP specification) is always run afterward +func (r resolverQuery) pnpResolve(specifier string, parentURL string, parentManifest *pnpData) (string, bool) { + // If specifier is a Node.js builtin, then + if BuiltInNodeModules[specifier] { + // Set resolved to specifier itself and return it + return specifier, true + } + + // Otherwise, if specifier starts with "/", "./", or "../", then + if strings.HasPrefix(specifier, "/") || strings.HasPrefix(specifier, "./") || strings.HasPrefix(specifier, "../") { + // Set resolved to NM_RESOLVE(specifier, parentURL) and return it + return specifier, true + } + + // Otherwise, + // Note: specifier is now a bare identifier + // Let unqualified be RESOLVE_TO_UNQUALIFIED(specifier, parentURL) + // Set resolved to NM_RESOLVE(unqualified, parentURL) + return r.resolveToUnqualified(specifier, parentURL, parentManifest) +} + +func parseBareIdentifier(specifier string) (ident string, modulePath string, ok bool) { + slash := strings.IndexByte(specifier, '/') + + // If specifier starts with "@", then + if strings.HasPrefix(specifier, "@") { + // If specifier doesn't contain a "/" separator, then + if slash == -1 { + // Throw an error + return + } + + // Otherwise, + // Set ident to the substring of specifier until the second "/" separator or the end of string, whatever happens first + if slash2 := strings.IndexByte(specifier[slash+1:], '/'); slash2 != -1 { + ident = specifier[:slash+1+slash2] + } else { + ident = specifier + } + } else { + // Otherwise, + // Set ident to the substring of specifier until the first "/" separator or the end of string, whatever happens first + if slash != -1 { + ident = specifier[:slash] + } else { + ident = specifier + } + } + + // Set modulePath to the substring of specifier starting from ident.length + modulePath = specifier[len(ident):] + + // Return {ident, modulePath} + ok = true + return +} + +func (r resolverQuery) resolveToUnqualified(specifier string, parentURL string, manifest *pnpData) (string, bool) { + // Let resolved be undefined + + // Let ident and modulePath be the result of PARSE_BARE_IDENTIFIER(specifier) + ident, modulePath, ok := parseBareIdentifier(specifier) + if !ok { + return "", false + } + + // Let manifest be FIND_PNP_MANIFEST(parentURL) + // (this is already done by the time we get here) + + // If manifest is null, then + // Set resolved to NM_RESOLVE(specifier, parentURL) and return it + if manifest == nil { + return specifier, true + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf("Using Yarn PnP manifest from %q to resolve %q", manifest.absPath, ident)) + } + + // Let parentLocator be FIND_LOCATOR(manifest, parentURL) + parentLocator, ok := r.findLocator(manifest, parentURL) + + // If parentLocator is null, then + // Set resolved to NM_RESOLVE(specifier, parentURL) and return it + if !ok { + return specifier, true + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Found parent locator: [%s, %s]", quoteOrNullIfEmpty(parentLocator.ident), quoteOrNullIfEmpty(parentLocator.reference))) + } + + // Let parentPkg be GET_PACKAGE(manifest, parentLocator) + parentPkg, ok := r.getPackage(manifest, parentLocator.ident, parentLocator.reference) + if !ok { + // We aren't supposed to get here according to the Yarn PnP specification + return "", false + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Found parent package at %q", parentPkg.packageLocation)) + } + + // Let referenceOrAlias be the entry from parentPkg.packageDependencies referenced by ident + referenceOrAlias, ok := parentPkg.packageDependencies[ident] + + // If referenceOrAlias is null or undefined, then + if !ok || referenceOrAlias.reference == "" { + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Failed to find %q in \"packageDependencies\" of parent package", ident)) + } + + // If manifest.enableTopLevelFallback is true, then + if manifest.enableTopLevelFallback { + if r.debugLogs != nil { + r.debugLogs.addNote(" Searching for a fallback because \"enableTopLevelFallback\" is true") + } + + // If parentLocator isn't in manifest.fallbackExclusionList, then + if set, _ := manifest.fallbackExclusionList[parentLocator.ident]; !set[parentLocator.reference] { + // Let fallback be RESOLVE_VIA_FALLBACK(manifest, ident) + fallback, _ := r.resolveViaFallback(manifest, ident) + + // If fallback is neither null nor undefined + if fallback.reference != "" { + // Set referenceOrAlias to fallback + referenceOrAlias = fallback + ok = true + } + } else if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Stopping because [%s, %s] is in \"fallbackExclusionList\"", + quoteOrNullIfEmpty(parentLocator.ident), quoteOrNullIfEmpty(parentLocator.reference))) + } + } + } + + // If referenceOrAlias is still undefined, then + if !ok { + // Throw a resolution error + return "", false + } + + // If referenceOrAlias is still null, then + if referenceOrAlias.reference == "" { + // Note: It means that parentPkg has an unfulfilled peer dependency on ident + // Throw a resolution error + return "", false + } + + if r.debugLogs != nil { + var referenceOrAliasStr string + if referenceOrAlias.ident != "" { + referenceOrAliasStr = fmt.Sprintf("[%q, %q]", referenceOrAlias.ident, referenceOrAlias.reference) + } else { + referenceOrAliasStr = quoteOrNullIfEmpty(referenceOrAlias.reference) + } + r.debugLogs.addNote(fmt.Sprintf(" Found dependency locator: [%s, %s]", quoteOrNullIfEmpty(ident), referenceOrAliasStr)) + } + + // Otherwise, if referenceOrAlias is an array, then + var dependencyPkg pnpPackage + if referenceOrAlias.ident != "" { + // Let alias be referenceOrAlias + alias := referenceOrAlias + + // Let dependencyPkg be GET_PACKAGE(manifest, alias) + dependencyPkg, ok = r.getPackage(manifest, alias.ident, alias.reference) + if !ok { + // We aren't supposed to get here according to the Yarn PnP specification + return "", false + } + } else { + // Otherwise, + // Let dependencyPkg be GET_PACKAGE(manifest, {ident, reference}) + dependencyPkg, ok = r.getPackage(manifest, ident, referenceOrAlias.reference) + if !ok { + // We aren't supposed to get here according to the Yarn PnP specification + return "", false + } + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Found package %q at %q", ident, dependencyPkg.packageLocation)) + } + + // Return dependencyPkg.packageLocation concatenated with modulePath + resolved := dependencyPkg.packageLocation + modulePath + result := r.fs.Join(manifest.absDirPath, resolved) + if strings.HasSuffix(resolved, "/") && !strings.HasSuffix(result, "/") { + result += "/" // This is important for matching Yarn PnP's expectations in tests + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Resolved %q via Yarn PnP to %q", specifier, result)) + } + return result, true +} + +func (r resolverQuery) findLocator(manifest *pnpData, moduleUrl string) (pnpIdentAndReference, bool) { + // Let relativeUrl be the relative path between manifest and moduleUrl + relativeUrl, ok := r.fs.Rel(manifest.absDirPath, moduleUrl) + if !ok { + return pnpIdentAndReference{}, false + } + + // The relative path must not start with ./; trim it if needed + if strings.HasPrefix(relativeUrl, "./") { + relativeUrl = relativeUrl[2:] + } + + // If relativeUrl matches manifest.ignorePatternData, then + if manifest.ignorePatternData != nil && manifest.ignorePatternData.MatchString(relativeUrl) { + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Ignoring %q because it matches \"ignorePatternData\"", relativeUrl)) + } + + // Return null + return pnpIdentAndReference{}, false + } + + // Note: Make sure relativeUrl always starts with a ./ or ../ + if !strings.HasSuffix(relativeUrl, "/") { + relativeUrl += "/" + } + if !strings.HasPrefix(relativeUrl, "./") && !strings.HasPrefix(relativeUrl, "../") { + relativeUrl = "./" + relativeUrl + } + + // This is the inner loop from Yarn's PnP resolver implementation. This is + // different from the specification, which contains a hypothetical slow + // algorithm instead. The algorithm from the specification can sometimes + // produce different results from the one used by the implementation, so + // we follow the implementation. + for { + entry, ok := manifest.packageLocatorsByLocations[relativeUrl] + if !ok || entry.discardFromLookup { + // Remove the last path component and try again + relativeUrl = relativeUrl[:strings.LastIndexByte(relativeUrl[:len(relativeUrl)-1], '/')+1] + if relativeUrl == "" { + break + } + continue + } + return entry.locator, true + } + + return pnpIdentAndReference{}, false +} + +func (r resolverQuery) resolveViaFallback(manifest *pnpData, ident string) (pnpIdentAndReference, bool) { + // Let topLevelPkg be GET_PACKAGE(manifest, {null, null}) + topLevelPkg, ok := r.getPackage(manifest, "", "") + if !ok { + // We aren't supposed to get here according to the Yarn PnP specification + return pnpIdentAndReference{}, false + } + + // Let referenceOrAlias be the entry from topLevelPkg.packageDependencies referenced by ident + referenceOrAlias, ok := topLevelPkg.packageDependencies[ident] + + // If referenceOrAlias is defined, then + if ok { + // Return it immediately + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf(" Found fallback for %q in \"packageDependencies\" of top-level package: [%s, %s]", ident, + quoteOrNullIfEmpty(referenceOrAlias.ident), quoteOrNullIfEmpty(referenceOrAlias.reference))) + } + return referenceOrAlias, true + } + + // Otherwise, + // Let referenceOrAlias be the entry from manifest.fallbackPool referenced by ident + referenceOrAlias, ok = manifest.fallbackPool[ident] + + // Return it immediatly, whether it's defined or not + if r.debugLogs != nil { + if ok { + r.debugLogs.addNote(fmt.Sprintf(" Found fallback for %q in \"fallbackPool\": [%s, %s]", ident, + quoteOrNullIfEmpty(referenceOrAlias.ident), quoteOrNullIfEmpty(referenceOrAlias.reference))) + } else { + r.debugLogs.addNote(fmt.Sprintf(" Failed to find fallback for %q in \"fallbackPool\"", ident)) + } + } + return referenceOrAlias, ok +} + +func (r resolverQuery) getPackage(manifest *pnpData, ident string, reference string) (pnpPackage, bool) { + if inner, ok := manifest.packageRegistryData[ident]; ok { + if pkg, ok := inner[reference]; ok { + return pkg, true + } + } + + if r.debugLogs != nil { + // We aren't supposed to get here according to the Yarn PnP specification: + // "Note: pkg cannot be undefined here; all packages referenced in any of the + // Plug'n'Play data tables MUST have a corresponding entry inside packageRegistryData." + r.debugLogs.addNote(fmt.Sprintf(" Yarn PnP invariant violation: GET_PACKAGE failed to find a package: [%s, %s]", + quoteOrNullIfEmpty(ident), quoteOrNullIfEmpty(reference))) + } + return pnpPackage{}, false +} + +func quoteOrNullIfEmpty(str string) string { + if str != "" { + return fmt.Sprintf("%q", str) + } + return "null" +} + +func compileYarnPnPData(absPath string, absDirPath string, json js_ast.Expr) *pnpData { + data := pnpData{ + absPath: absPath, + absDirPath: absDirPath, + } + + if value, _, ok := getProperty(json, "dependencyTreeRoots"); ok { + if array, ok := value.Data.(*js_ast.EArray); ok { + data.dependencyTreeRoots = make(map[string]string, len(array.Items)) + + for _, item := range array.Items { + if name, _, ok := getProperty(item, "name"); ok { + if reference, _, ok := getProperty(item, "reference"); ok { + if name, ok := getString(name); ok { + if reference, ok := getString(reference); ok { + data.dependencyTreeRoots[name] = reference + } + } + } + } + } + } + } + + if value, _, ok := getProperty(json, "enableTopLevelFallback"); ok { + if enableTopLevelFallback, ok := getBool(value); ok { + data.enableTopLevelFallback = enableTopLevelFallback + } + } + + if value, _, ok := getProperty(json, "fallbackExclusionList"); ok { + if array, ok := value.Data.(*js_ast.EArray); ok { + data.fallbackExclusionList = make(map[string]map[string]bool, len(array.Items)) + + for _, item := range array.Items { + if tuple, ok := item.Data.(*js_ast.EArray); ok && len(tuple.Items) == 2 { + if ident, ok := getStringOrNull(tuple.Items[0]); ok { + if array2, ok := tuple.Items[1].Data.(*js_ast.EArray); ok { + references := make(map[string]bool, len(array2.Items)) + + for _, item2 := range array2.Items { + if reference, ok := getString(item2); ok { + references[reference] = true + } + } + + data.fallbackExclusionList[ident] = references + } + } + } + } + } + } + + if value, _, ok := getProperty(json, "fallbackPool"); ok { + if array, ok := value.Data.(*js_ast.EArray); ok { + data.fallbackPool = make(map[string]pnpIdentAndReference, len(array.Items)) + + for _, item := range array.Items { + if array2, ok := item.Data.(*js_ast.EArray); ok && len(array2.Items) == 2 { + if ident, ok := getString(array2.Items[0]); ok { + if dependencyTarget, ok := getDependencyTarget(array2.Items[1]); ok { + data.fallbackPool[ident] = dependencyTarget + } + } + } + } + } + } + + if value, _, ok := getProperty(json, "ignorePatternData"); ok { + if ignorePatternData, ok := getString(value); ok { + data.ignorePatternData, _ = regexp.Compile(ignorePatternData) + } + } + + if value, _, ok := getProperty(json, "packageRegistryData"); ok { + if array, ok := value.Data.(*js_ast.EArray); ok { + data.packageRegistryData = make(map[string]map[string]pnpPackage, len(array.Items)) + data.packageLocatorsByLocations = make(map[string]pnpPackageLocatorByLocation) + + for _, item := range array.Items { + if tuple, ok := item.Data.(*js_ast.EArray); ok && len(tuple.Items) == 2 { + if packageIdent, ok := getStringOrNull(tuple.Items[0]); ok { + if array2, ok := tuple.Items[1].Data.(*js_ast.EArray); ok { + references := make(map[string]pnpPackage, len(array2.Items)) + data.packageRegistryData[packageIdent] = references + + for _, item2 := range array2.Items { + if tuple2, ok := item2.Data.(*js_ast.EArray); ok && len(tuple2.Items) == 2 { + if packageReference, ok := getStringOrNull(tuple2.Items[0]); ok { + pkg := tuple2.Items[1] + + if packageLocation, _, ok := getProperty(pkg, "packageLocation"); ok { + if packageDependencies, _, ok := getProperty(pkg, "packageDependencies"); ok { + if packageLocation, ok := getString(packageLocation); ok { + if array3, ok := packageDependencies.Data.(*js_ast.EArray); ok { + deps := make(map[string]pnpIdentAndReference, len(array3.Items)) + discardFromLookup := false + + for _, dep := range array3.Items { + if array4, ok := dep.Data.(*js_ast.EArray); ok && len(array4.Items) == 2 { + if ident, ok := getString(array4.Items[0]); ok { + if dependencyTarget, ok := getDependencyTarget(array4.Items[1]); ok { + deps[ident] = dependencyTarget + } + } + } + } + + if value, _, ok := getProperty(pkg, "discardFromLookup"); ok { + if value, ok := getBool(value); ok { + discardFromLookup = value + } + } + + references[packageReference] = pnpPackage{ + packageLocation: packageLocation, + packageDependencies: deps, + discardFromLookup: discardFromLookup, + } + + // This is what Yarn's PnP implementation does (specifically in + // "hydrateRuntimeState"), so we replicate that behavior here: + if entry, ok := data.packageLocatorsByLocations[packageLocation]; !ok { + data.packageLocatorsByLocations[packageLocation] = pnpPackageLocatorByLocation{ + locator: pnpIdentAndReference{ident: packageIdent, reference: packageReference}, + discardFromLookup: discardFromLookup, + } + } else { + entry.discardFromLookup = entry.discardFromLookup && discardFromLookup + if !discardFromLookup { + entry.locator = pnpIdentAndReference{ident: packageIdent, reference: packageReference} + } + data.packageLocatorsByLocations[packageLocation] = entry + } + } + } + } + } + } + } + } + } + } + } + } + } + } + + return &data +} + +func getStringOrNull(json js_ast.Expr) (string, bool) { + switch value := json.Data.(type) { + case *js_ast.EString: + return helpers.UTF16ToString(value.Value), true + + case *js_ast.ENull: + return "", true + } + + return "", false +} + +func getDependencyTarget(json js_ast.Expr) (pnpIdentAndReference, bool) { + switch d := json.Data.(type) { + case *js_ast.ENull: + return pnpIdentAndReference{}, true + + case *js_ast.EString: + return pnpIdentAndReference{reference: helpers.UTF16ToString(d.Value)}, true + + case *js_ast.EArray: + if len(d.Items) == 2 { + if name, ok := getString(d.Items[0]); ok { + if reference, ok := getString(d.Items[1]); ok { + return pnpIdentAndReference{ + ident: name, + reference: reference, + }, true + } + } + } + } + + return pnpIdentAndReference{}, false +} + +func (r resolverQuery) extractYarnPnPDataFromJSON(pnpDataPath string, jsonCache *cache.JSONCache) (result js_ast.Expr) { + contents, err, originalError := r.caches.FSCache.ReadFile(r.fs, pnpDataPath) + if r.debugLogs != nil && originalError != nil { + r.debugLogs.addNote(fmt.Sprintf("Failed to read file %q: %s", pnpDataPath, originalError.Error())) + } + if err != nil { + r.log.AddError(nil, logger.Range{}, + fmt.Sprintf("Cannot read file %q: %s", + r.PrettyPath(logger.Path{Text: pnpDataPath, Namespace: "file"}), err.Error())) + return + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf("The file %q exists", pnpDataPath)) + } + keyPath := logger.Path{Text: pnpDataPath, Namespace: "file"} + source := logger.Source{ + KeyPath: keyPath, + PrettyPath: r.PrettyPath(keyPath), + Contents: contents, + } + result, _ = jsonCache.Parse(r.log, source, js_parser.JSONOptions{}) + return +} + +func (r resolverQuery) tryToExtractYarnPnPDataFromJS(pnpDataPath string, jsonCache *cache.JSONCache) (result js_ast.Expr) { + contents, err, originalError := r.caches.FSCache.ReadFile(r.fs, pnpDataPath) + if r.debugLogs != nil && originalError != nil { + r.debugLogs.addNote(fmt.Sprintf("Failed to read file %q: %s", pnpDataPath, originalError.Error())) + } + if err != nil { + r.log.AddError(nil, logger.Range{}, + fmt.Sprintf("Cannot read file %q: %s", + r.PrettyPath(logger.Path{Text: pnpDataPath, Namespace: "file"}), err.Error())) + return + } + if r.debugLogs != nil { + r.debugLogs.addNote(fmt.Sprintf("The file %q exists", pnpDataPath)) + } + + keyPath := logger.Path{Text: pnpDataPath, Namespace: "file"} + source := logger.Source{ + KeyPath: keyPath, + PrettyPath: r.PrettyPath(keyPath), + Contents: contents, + } + ast, _ := js_parser.Parse(r.log, source, js_parser.OptionsForYarnPnP()) + + if r.debugLogs != nil && ast.ManifestForYarnPnP.Data != nil { + r.debugLogs.addNote(fmt.Sprintf(" Extracted JSON data from %q", pnpDataPath)) + } + return ast.ManifestForYarnPnP +} diff --git a/internal/resolver/yarnpnp_test.go b/internal/resolver/yarnpnp_test.go new file mode 100644 index 00000000000..df1c5c4b312 --- /dev/null +++ b/internal/resolver/yarnpnp_test.go @@ -0,0 +1,90 @@ +package resolver + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "testing" + + "github.com/evanw/esbuild/internal/config" + "github.com/evanw/esbuild/internal/fs" + "github.com/evanw/esbuild/internal/js_parser" + "github.com/evanw/esbuild/internal/logger" + "github.com/evanw/esbuild/internal/test" +) + +type pnpTestExpectation struct { + Manifest interface{} + Tests []pnpTest +} + +type pnpTest struct { + It string + Imported string + Importer string + Expected string +} + +func TestYarnPnP(t *testing.T) { + t.Helper() + contents, err := ioutil.ReadFile("testExpectations.json") + if err != nil { + t.Fatalf("Failed to read testExpectations.json: %s", err.Error()) + } + + var expectations []pnpTestExpectation + err = json.Unmarshal(contents, &expectations) + if err != nil { + t.Fatalf("Failed to parse testExpectations.json: %s", err.Error()) + } + + for i, expectation := range expectations { + path := fmt.Sprintf("testExpectations[%d].manifest", i) + contents, err := json.Marshal(expectation.Manifest) + if err != nil { + t.Fatalf("Failed to generate JSON: %s", err.Error()) + } + + source := logger.Source{ + KeyPath: logger.Path{Text: path}, + PrettyPath: path, + Contents: string(contents), + } + tempLog := logger.NewDeferLog(logger.DeferLogAll, nil) + expr, ok := js_parser.ParseJSON(tempLog, source, js_parser.JSONOptions{}) + if !ok { + t.Fatalf("Failed to re-parse JSON: %s", path) + } + + msgs := tempLog.Done() + if len(msgs) != 0 { + t.Fatalf("Log not empty after re-parsing JSON: %s", path) + } + + manifest := compileYarnPnPData(path, "/path/to/project/", expr) + + for _, current := range expectation.Tests { + func(current pnpTest) { + t.Run(current.It, func(t *testing.T) { + rr := NewResolver(fs.MockFS(nil), logger.NewDeferLog(logger.DeferLogNoVerboseOrDebug, nil), nil, config.Options{}) + r := resolverQuery{resolver: rr.(*resolver)} + result, ok := r.pnpResolve(current.Imported, current.Importer, manifest) + if !ok { + result = "error!" + } + expected := current.Expected + + // If a we aren't going through PnP, then we should just run the + // normal node module resolution rules instead of throwing an error. + // However, this test requires us to throw an error, which seems + // incorrect. So we change the expected value of the test instead. + if current.It == `shouldn't go through PnP when trying to resolve dependencies from packages covered by ignorePatternData` { + expected = current.Imported + } + + test.AssertEqualWithDiff(t, result, expected) + }) + }(current) + } + } +} diff --git a/scripts/js-api-tests.js b/scripts/js-api-tests.js index 3f8be7b568a..b9eed43e217 100644 --- a/scripts/js-api-tests.js +++ b/scripts/js-api-tests.js @@ -2711,6 +2711,260 @@ require("/assets/file.png"); virtual3: foo_default4 }); })(); +`) + }, + + async yarnPnP_pnp_data_json({ esbuild, testDir }) { + const entry = path.join(testDir, 'entry.js') + const manifest = path.join(testDir, '.pnp.data.json') + const leftPad = path.join(testDir, '.yarn', 'cache', 'left-pad', 'index.js') + + await writeFileAsync(entry, ` + import leftPad from 'left-pad' + console.log(leftPad()) + `) + + await writeFileAsync(manifest, `{ + "packageRegistryData": [ + [null, [ + [null, { + "packageLocation": "./", + "packageDependencies": [ + ["left-pad", "npm:1.3.0"] + ], + "linkType": "SOFT" + }] + ]], + ["left-pad", [ + ["npm:1.3.0", { + "packageLocation": "./.yarn/cache/left-pad/", + "packageDependencies": [ + ["left-pad", "npm:1.3.0"] + ], + "linkType": "HARD" + }] + ]] + ] + }`) + + await mkdirAsync(path.dirname(leftPad), { recursive: true }) + await writeFileAsync(leftPad, `export default function() {}`) + + const value = await esbuild.build({ + entryPoints: [entry], + bundle: true, + write: false, + }) + + assert.strictEqual(value.outputFiles.length, 1) + assert.strictEqual(value.outputFiles[0].text, `(() => { + // scripts/.js-api-tests/yarnPnP_pnp_data_json/.yarn/cache/left-pad/index.js + function left_pad_default() { + } + + // scripts/.js-api-tests/yarnPnP_pnp_data_json/entry.js + console.log(left_pad_default()); +})(); +`) + }, + + async yarnPnP_pnp_js_object_literal({ esbuild, testDir }) { + const entry = path.join(testDir, 'entry.js') + const manifest = path.join(testDir, '.pnp.js') + const leftPad = path.join(testDir, '.yarn', 'cache', 'left-pad', 'index.js') + + await writeFileAsync(entry, ` + import leftPad from 'left-pad' + console.log(leftPad()) + `) + + await writeFileAsync(manifest, `#!/usr/bin/env node + /* eslint-disable */ + + try { + Object.freeze({}).detectStrictMode = true; + } catch (error) { + throw new Error(); + } + + function $$SETUP_STATE(hydrateRuntimeState, basePath) { + return hydrateRuntimeState({ + "packageRegistryData": [ + [null, [ + [null, { + "packageLocation": "./", + "packageDependencies": [ + ["left-pad", "npm:1.3.0"] + ], + "linkType": "SOFT" + }] + ]], + ["left-pad", [ + ["npm:1.3.0", { + "packageLocation": "./.yarn/cache/left-pad/", + "packageDependencies": [ + ["left-pad", "npm:1.3.0"] + ], + "linkType": "HARD" + }] + ]] + ] + }) + } + `) + + await mkdirAsync(path.dirname(leftPad), { recursive: true }) + await writeFileAsync(leftPad, `export default function() {}`) + + const value = await esbuild.build({ + entryPoints: [entry], + bundle: true, + write: false, + }) + + assert.strictEqual(value.outputFiles.length, 1) + assert.strictEqual(value.outputFiles[0].text, `(() => { + // scripts/.js-api-tests/yarnPnP_pnp_js_object_literal/.yarn/cache/left-pad/index.js + function left_pad_default() { + } + + // scripts/.js-api-tests/yarnPnP_pnp_js_object_literal/entry.js + console.log(left_pad_default()); +})(); +`) + }, + + async yarnPnP_pnp_cjs_JSON_parse_string_literal({ esbuild, testDir }) { + const entry = path.join(testDir, 'entry.js') + const manifest = path.join(testDir, '.pnp.cjs') + const leftPad = path.join(testDir, '.yarn', 'cache', 'left-pad', 'index.js') + + await writeFileAsync(entry, ` + import leftPad from 'left-pad' + console.log(leftPad()) + `) + + await writeFileAsync(manifest, `#!/usr/bin/env node + /* eslint-disable */ + + try { + Object.freeze({}).detectStrictMode = true; + } catch (error) { + throw new Error(); + } + + function $$SETUP_STATE(hydrateRuntimeState, basePath) { + return hydrateRuntimeState(JSON.parse('{\\ + "packageRegistryData": [\\ + [null, [\\ + [null, {\\ + "packageLocation": "./",\\ + "packageDependencies": [\\ + ["left-pad", "npm:1.3.0"]\\ + ],\\ + "linkType": "SOFT"\\ + }]\\ + ]],\\ + ["left-pad", [\\ + ["npm:1.3.0", {\\ + "packageLocation": "./.yarn/cache/left-pad/",\\ + "packageDependencies": [\\ + ["left-pad", "npm:1.3.0"]\\ + ],\\ + "linkType": "HARD"\\ + }]\\ + ]]\\ + ]\\ + }')) + } + `) + + await mkdirAsync(path.dirname(leftPad), { recursive: true }) + await writeFileAsync(leftPad, `export default function() {}`) + + const value = await esbuild.build({ + entryPoints: [entry], + bundle: true, + write: false, + }) + + assert.strictEqual(value.outputFiles.length, 1) + assert.strictEqual(value.outputFiles[0].text, `(() => { + // scripts/.js-api-tests/yarnPnP_pnp_cjs_JSON_parse_string_literal/.yarn/cache/left-pad/index.js + function left_pad_default() { + } + + // scripts/.js-api-tests/yarnPnP_pnp_cjs_JSON_parse_string_literal/entry.js + console.log(left_pad_default()); +})(); +`) + }, + + async yarnPnP_pnp_cjs_JSON_parse_identifier({ esbuild, testDir }) { + const entry = path.join(testDir, 'entry.js') + const manifest = path.join(testDir, '.pnp.cjs') + const leftPad = path.join(testDir, '.yarn', 'cache', 'left-pad', 'index.js') + + await writeFileAsync(entry, ` + import leftPad from 'left-pad' + console.log(leftPad()) + `) + + await writeFileAsync(manifest, `#!/usr/bin/env node + /* eslint-disable */ + + try { + Object.freeze({}).detectStrictMode = true; + } catch (error) { + throw new Error(); + } + + const RAW_RUNTIME_STATE = '{\\ + "packageRegistryData": [\\ + [null, [\\ + [null, {\\ + "packageLocation": "./",\\ + "packageDependencies": [\\ + ["left-pad", "npm:1.3.0"]\\ + ],\\ + "linkType": "SOFT"\\ + }]\\ + ]],\\ + ["left-pad", [\\ + ["npm:1.3.0", {\\ + "packageLocation": "./.yarn/cache/left-pad/",\\ + "packageDependencies": [\\ + ["left-pad", "npm:1.3.0"]\\ + ],\\ + "linkType": "HARD"\\ + }]\\ + ]]\\ + ]\\ + }' + + function $$SETUP_STATE(hydrateRuntimeState, basePath) { + return hydrateRuntimeState(JSON.parse(RAW_RUNTIME_STATE)) + } + `) + + await mkdirAsync(path.dirname(leftPad), { recursive: true }) + await writeFileAsync(leftPad, `export default function() {}`) + + const value = await esbuild.build({ + entryPoints: [entry], + bundle: true, + write: false, + }) + + assert.strictEqual(value.outputFiles.length, 1) + assert.strictEqual(value.outputFiles[0].text, `(() => { + // scripts/.js-api-tests/yarnPnP_pnp_cjs_JSON_parse_identifier/.yarn/cache/left-pad/index.js + function left_pad_default() { + } + + // scripts/.js-api-tests/yarnPnP_pnp_cjs_JSON_parse_identifier/entry.js + console.log(left_pad_default()); +})(); `) }, } From 3d0b0b4e428cff9a4d03e4a76be46da9b183390e Mon Sep 17 00:00:00 2001 From: Evan Wallace Date: Tue, 9 Aug 2022 22:31:22 -0400 Subject: [PATCH 2/2] release notes --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 201a4389439..6af330e1f5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## Unreleased + +**This release contains backwards-incompatible changes.** Since esbuild is before version 1.0.0, these changes have been released as a new minor version to reflect this (as [recommended by npm](https://docs.npmjs.com/cli/v6/using-npm/semver/)). You should either be pinning the exact version of `esbuild` in your `package.json` file or be using a version range syntax that only accepts patch upgrades such as `~0.14.0`. See the documentation about [semver](https://docs.npmjs.com/cli/v6/using-npm/semver/) for more information. + +* Implement the Yarn Plug'n'Play module resolution algorithm ([#154](https://github.com/evanw/esbuild/issues/154), [#237](https://github.com/evanw/esbuild/issues/237), [#1263](https://github.com/evanw/esbuild/issues/1263), [#2451](https://github.com/evanw/esbuild/pull/2451)) + + [Node](https://nodejs.org/) comes with a package manager called [npm](https://www.npmjs.com/), which installs packages into a `node_modules` folder. Node and esbuild both come with built-in rules for resolving import paths to packages within `node_modules`, so packages installed via npm work automatically without any configuration. However, many people use an alternative package manager called [Yarn](https://yarnpkg.com/). While Yarn can install packages using `node_modules`, it also offers a different package installation strategy called [Plug'n'Play](https://yarnpkg.com/features/pnp/), which is often shortened to "PnP" (not to be confused with [pnpm](https://pnpm.io/), which is an entirely different unrelated package manager). + + Plug'n'Play installs packages as `.zip` files on your file system. The packages are never actually unzipped. Since Node doesn't know anything about Yarn's package installation strategy, this means you can no longer run your code with Node as it won't be able to find your packages. Instead, you need to run your code with Yarn, which applies patches to Node's file system APIs before running your code. These patches attempt to make zip files seem like normal directories. When running under Yarn, using Node's file system API to read `./some.zip/lib/file.js` actually automatically extracts `lib/file.js` from `./some.zip` at run-time as if it was a normal file. Other file system APIs behave similarly. However, these patches don't work with esbuild because esbuild is not written in JavaScript; it's a native binary executable that interacts with the file system directly through the operating system. + + Previously the workaround for using esbuild with Plug'n'Play was to use the [`@yarnpkg/esbuild-plugin-pnp`](https://www.npmjs.com/package/@yarnpkg/esbuild-plugin-pnp) plugin with esbuild's JavaScript API. However, this wasn't great because the plugin needed to potentially intercept every single import path and file load to check whether it was a Plug'n'Play package, which has an unusually high performance cost. It also meant that certain subtleties of path resolution rules within a `.zip` file could differ slightly from the way esbuild normally works since path resolution inside `.zip` files was implemented by Yarn, not by esbuild (which is due to a limitation of esbuild's plugin API). + + With this release, esbuild now contains an independent implementation of Yarn's Plug'n'Play algorithm (which is used when esbuild finds a `.pnp.js`, `.pnp.cjs`, or `.pnp.data.json` file in the directory tree). Creating additional implementations of this algorithm recently became possible because Yarn's package manifest format was recently documented: https://yarnpkg.com/advanced/pnp-spec/. This should mean that you can now use esbuild to bundle Plug'n'Play projects without any additional configuration (so you shouldn't need `@yarnpkg/esbuild-plugin-pnp` anymore). Bundling these projects should now happen much faster as Yarn no longer even needs to be run at all. And path resolution rules within Yarn packages should now be consistent with how esbuild handles regular Node packages. For example, fields such as `module` and `browser` in `package.json` files within `.zip` files should now be respected. + + Keep in mind that this is brand new code and there may be some initial issues to work through before esbuild's implementation is solid. Yarn's Plug'n'Play specification is also brand new and may need some follow-up edits to guide new implementations to match Yarn's exact behavior. If you try this out, make sure to test it before committing to using it, and let me know if anything isn't working as expected. Should you need to debug esbuild's path resolution, you may find `--log-level=verbose` helpful. + ## 0.14.54 * Fix optimizations for calls containing spread arguments ([#2445](https://github.com/evanw/esbuild/issues/2445))