Use "real modules" for ember-source when possible (3.27+)

.vscode/launch.json

@@ -10,13 +10,7 @@
       "name": "Run tests",
       "program": "${workspaceFolder}/node_modules/.bin/jest",
       "cwd": "${workspaceFolder}/packages/core",
-      "args": [
-        "--runInBand",
-        "--testPathPattern",
-        "tests/portable-babel-config.test.js",
-        "--test-name-pattern",
-        "undefined"
-      ],
+      "args": ["--runInBand", "--testPathPattern", "tests/import-adder.test.js"],
       "outputCapture": "std"
     },
     {
@@ -143,7 +137,8 @@
       "cwd": "${workspaceFolder}/packages/util",
       "args": ["build"],
       "env": {
-        "JOBS": "1"
+        "JOBS": "1",
+        "EMBROIDER_TEST_SETUP_FORCE": "embroider"
       },
       "outputCapture": "std"
     },

package.json

@@ -34,7 +34,8 @@
   },
   "resolutions": {
     "**/browserslist": "^4.14.0",
-    "**/fastboot": "^3.1.0"
+    "**/fastboot": "^3.1.0",
+    "**/qunit": "^2.14.1"
   },
   "devDependencies": {
     "@types/jest": "^24.0.11",
@@ -81,7 +82,7 @@
   },
   "volta": {
     "node": "12.16.1",
-    "yarn": "1.17.3"
+    "yarn": "1.22.5"
   },
   "version": "0.37.0"
 }

packages/compat/src/compat-adapters/ember-source.ts

@@ -0,0 +1,76 @@
+import V1Addon from '../v1-addon';
+import Funnel from 'broccoli-funnel';
+import mergeTrees from 'broccoli-merge-trees';
+import AddToTree from '../add-to-tree';
+import { outputFileSync, unlinkSync } from 'fs-extra';
+import { join } from 'path';
+import semver from 'semver';
+
+export default class extends V1Addon {
+  private useRealModules = semver.satisfies(this.packageJSON.version, '>=3.27.0', { includePrerelease: true });
+
+  // when using real modules, we're replacing treeForAddon and treeForVendor
+  customizes(treeName: string) {
+    return (
+      (this.useRealModules && (treeName === 'treeForAddon' || treeName === 'treeForVendor')) ||
+      super.customizes(treeName)
+    );
+  }
+
+  invokeOriginalTreeFor(name: string, opts: { neuterPreprocessors: boolean } = { neuterPreprocessors: false }) {
+    if (this.useRealModules) {
+      if (name === 'addon') {
+        return this.customAddonTree();
+      }
+      if (name === 'vendor') {
+        return this.customVendorTree();
+      }
+    }
+    return super.invokeOriginalTreeFor(name, opts);
+  }
+
+  // Our addon tree is all of the "packages" we share. @embroider/compat already
+  // supports that pattern of emitting modules into other package's namespaces.
+  private customAddonTree() {
+    return mergeTrees([
+      new Funnel(this.rootTree, {
+        srcDir: 'dist/packages',
+      }),
+      new Funnel(this.rootTree, {
+        srcDir: 'dist/dependencies',
+      }),
+    ]);
+  }
+
+  // We're zeroing out these files in vendor rather than deleting them, because
+  // we can't easily intercept the `app.import` that presumably exists for them,
+  // so rather than error they will just be empty.
+  //
+  // The reason we're zeroing these out is that we're going to consume all our
+  // modules directly out of treeForAddon instead, as real modules that webpack
+  // can see.
+  private customVendorTree() {
+    return new AddToTree(this.addonInstance._treeFor('vendor'), outputPath => {
+      unlinkSync(join(outputPath, 'ember', 'ember.js'));
+      outputFileSync(join(outputPath, 'ember', 'ember.js'), '');
+      unlinkSync(join(outputPath, 'ember', 'ember-testing.js'));
+      outputFileSync(join(outputPath, 'ember', 'ember-testing.js'), '');
+    });
+  }
+
+  get packageMeta() {
+    let meta = super.packageMeta;
+    if (this.useRealModules) {
+      if (!meta['implicit-modules']) {
+        meta['implicit-modules'] = [];
+      }
+      meta['implicit-modules'].push('./ember/index.js');
+
+      if (!meta['implicit-test-modules']) {
+        meta['implicit-test-modules'] = [];
+      }
+      meta['implicit-test-modules'].push('./ember-testing/index.js');
+    }
+    return meta;
+  }
+}

packages/compat/src/compat-app.ts

@@ -35,6 +35,7 @@ import bind from 'bind-decorator';
 import { pathExistsSync } from 'fs-extra';
 import { tmpdir } from 'os';
 import { Options as AdjustImportsOptions } from '@embroider/core/src/babel-plugin-adjust-imports';
+import semver from 'semver';
 
 interface TreeNames {
   appJS: BroccoliNode;
@@ -359,6 +360,9 @@ class CompatAppAdapter implements AppAdapter<TreeNames> {
       activeAddons[addon.name] = addon.root;
     }
 
+    let emberSource = this.activeAddonChildren().find(a => a.name === 'ember-source')!;
+    let emberNeedsModulesPolyfill = semver.satisfies(emberSource.version, '<3.27.0', { includePrerelease: true });
+
     return {
       activeAddons,
       renameModules,
@@ -374,6 +378,7 @@ class CompatAppAdapter implements AppAdapter<TreeNames> {
       // up as a side-effect of babel transpilation, and babel is subject to
       // persistent caching.
       externalsDir: join(tmpdir(), 'embroider', 'externals'),
+      emberNeedsModulesPolyfill,
     };
   }
 

packages/compat/src/default-pipeline.ts

@@ -4,7 +4,7 @@ import { Node } from 'broccoli-node-api';
 import writeFile from 'broccoli-file-creator';
 import mergeTrees from 'broccoli-merge-trees';
 
-interface PipelineOptions<PackagerOptions> extends Options {
+export interface PipelineOptions<PackagerOptions> extends Options {
   packagerOptions?: PackagerOptions;
   onOutputPath?: (outputPath: string) => void;
   variants?: Variant[];

packages/compat/src/index.ts

@@ -3,5 +3,5 @@ export { default as Addons } from './compat-addons';
 export { default as PrebuiltAddons } from './prebuilt-addons';
 export { default as Options, recommendedOptions } from './options';
 export { default as V1Addon } from './v1-addon';
-export { default as compatBuild } from './default-pipeline';
+export { default as compatBuild, PipelineOptions } from './default-pipeline';
 export { PackageRules, ModuleRules } from './dependency-rules';

packages/compat/src/resolver.ts

@@ -426,6 +426,10 @@ export default class CompatResolver implements Resolver {
     return null;
   }
 
+  get adjustImportsOptions() {
+    return this.params.adjustImportsOptions;
+  }
+
   @Memoize()
   private get appPackage(): AppPackagePlaceholder {
     return { root: this.params.root, name: this.params.modulePrefix };

packages/compat/src/synthesize-template-only-components.ts

@@ -4,7 +4,8 @@ import { join, basename } from 'path';
 import walkSync from 'walk-sync';
 import { remove, outputFileSync, pathExistsSync } from 'fs-extra';
 
-const source = `export default Ember._templateOnlyComponent();`;
+const source = `import templateOnlyComponent from '@ember/component/template-only';
+export default templateOnlyComponent();`;
 
 const templateExtension = '.hbs';
 

packages/compat/src/v1-addon.ts

@@ -101,6 +101,9 @@ class V1AddonCompatResolver implements Resolver {
       .replace(extensionsPattern(['.js', '.hbs']), '')
       .replace(/\/index$/, '');
   }
+  get adjustImportsOptions(): Resolver['adjustImportsOptions'] {
+    throw new Error(`bug: the addon compat resolver only supports absPath mapping`);
+  }
 }
 
 // This controls and types the interface between our new world and the classic

packages/compat/tests/audit.test.ts

@@ -39,6 +39,7 @@ describe('audit', function () {
             activeAddons: {},
             relocatedFiles: {},
             resolvableExtensions,
+            emberNeedsModulesPolyfill: true,
           },
         }),
       },

packages/compat/tests/resolver.test.ts

@@ -40,11 +40,17 @@ describe('compat-resolver', function () {
           activeAddons: {},
           relocatedFiles: {},
           resolvableExtensions: ['.js', '.hbs'],
+          emberNeedsModulesPolyfill: false,
         },
         otherOptions.adjustImportsImports
       ),
     });
-    let compiler = new TemplateCompiler({ compilerPath, resolver, EmberENV, plugins });
+    let compiler = new TemplateCompiler({
+      compilerPath,
+      resolver,
+      EmberENV,
+      plugins,
+    });
     return function (relativePath: string, contents: string) {
       let moduleName = givenFile(relativePath);
       let { dependencies } = compiler.precompile(moduleName, contents);

packages/compat/tests/template-colocation.test.ts

@@ -80,17 +80,19 @@ describe('template colocation', function () {
   test(`app's colocated template is associated with JS`, function () {
     let assertFile = expectFile('components/has-colocated-template.js').transform(build.transpile);
     assertFile.matches(/import TEMPLATE from ['"]\.\/has-colocated-template.hbs['"];/, 'imported template');
+    assertFile.matches(/const setComponentTemplate = Ember\._setComponentTemplate;/, 'found setComponentTemplate');
     assertFile.matches(
-      /export default Ember._setComponentTemplate\(TEMPLATE, class extends Component \{\}/,
+      /export default setComponentTemplate\(TEMPLATE, class extends Component \{\}/,
       'default export is wrapped'
     );
   });
 
   test(`app's template-only component JS is synthesized`, function () {
     let assertFile = expectFile('components/template-only-component.js').transform(build.transpile);
     assertFile.matches(/import TEMPLATE from ['"]\.\/template-only-component.hbs['"];/, 'imported template');
+    assertFile.matches(/const setComponentTemplate = Ember\._setComponentTemplate;/, 'found setComponentTemplate');
     assertFile.matches(
-      /export default Ember._setComponentTemplate\(TEMPLATE, Ember._templateOnlyComponent\(\)\)/,
+      /export default setComponentTemplate\(TEMPLATE, Ember._templateOnlyComponent\(\)\)/,
       'default export is wrapped'
     );
   });
@@ -108,17 +110,19 @@ describe('template colocation', function () {
   test(`addon's colocated template is associated with JS`, function () {
     let assertFile = expectFile('node_modules/my-addon/components/component-one.js').transform(build.transpile);
     assertFile.matches(/import TEMPLATE from ['"]\.\/component-one.hbs['"];/, 'imported template');
+    assertFile.matches(/const setComponentTemplate = Ember\._setComponentTemplate;/, 'found setComponentTemplate');
     assertFile.matches(
-      /export default Ember._setComponentTemplate\(TEMPLATE, class extends Component \{\}/,
+      /export default setComponentTemplate\(TEMPLATE, class extends Component \{\}/,
       'default export is wrapped'
     );
   });
 
   test(`addon's template-only component JS is synthesized`, function () {
     let assertFile = expectFile('node_modules/my-addon/components/component-two.js').transform(build.transpile);
     assertFile.matches(/import TEMPLATE from ['"]\.\/component-two.hbs['"];/, 'imported template');
+    assertFile.matches(/const setComponentTemplate = Ember\._setComponentTemplate;/, 'found setComponentTemplate');
     assertFile.matches(
-      /export default Ember._setComponentTemplate\(TEMPLATE, Ember._templateOnlyComponent\(\)\)/,
+      /export default setComponentTemplate\(TEMPLATE, Ember._templateOnlyComponent\(\)\)/,
       'default export is wrapped'
     );
   });

packages/core/src/app.ts

@@ -365,13 +365,13 @@ export class AppBuilder<TreeNames> {
       } as InlineBabelParams,
     ]);
 
-    babel.plugins.push(this.adjustImportsPlugin(appFiles));
-
     // this is @embroider/macros configured for full stage3 resolution
     babel.plugins.push(this.macrosConfig.babelPluginConfig());
 
     babel.plugins.push([require.resolve('./template-colocation-plugin')]);
 
+    babel.plugins.push(this.adjustImportsPlugin(appFiles));
+
     // we can use globally shared babel runtime by default
     babel.plugins.push([
       require.resolve('@babel/plugin-transform-runtime'),
@@ -1063,6 +1063,13 @@ export class AppBuilder<TreeNames> {
     }
 
     let eagerModules = [];
+    if (!this.adapter.adjustImportsOptions().emberNeedsModulesPolyfill) {
+      // when we're running with fake ember modules, vendor.js takes care of
+      // this bootstrapping. But when we're running with real ember modules,
+      // it's up to our entrypoint.
+      eagerModules.push('@ember/-internals/bootstrap');
+    }
+
     let requiredAppFiles = [this.requiredOtherFiles(appFiles)];
     if (!this.options.staticComponents) {
       requiredAppFiles.push(appFiles.components);
@@ -1207,6 +1214,13 @@ export class AppBuilder<TreeNames> {
       explicitRelative(dirname(myName), this.topAppJSAsset(engines, prepared).relativePath),
     ];
 
+    if (!this.adapter.adjustImportsOptions().emberNeedsModulesPolyfill) {
+      // when we're running with fake ember modules, the prebuilt test-support
+      // script takes care of this bootstrapping. But when we're running with
+      // real ember modules, it's up to our entrypoint.
+      eagerModules.push('ember-testing');
+    }
+
     let amdModules: { runtime: string; buildtime: string }[] = [];
     // this is a backward-compatibility feature: addons can force inclusion of
     // test support modules.

packages/core/src/babel-import-adder.ts

@@ -0,0 +1,89 @@
+import type { NodePath } from '@babel/traverse';
+import type * as t from '@babel/types';
+
+type BabelTypes = typeof t;
+
+export class ImportAdder {
+  constructor(private t: BabelTypes, private program: NodePath<t.Program>) {}
+
+  import(target: NodePath<t.Node>, moduleSpecifier: string, exportedName: string, nameHint?: string): t.Identifier {
+    let declaration = this.program
+      .get('body')
+      .find(elt => elt.isImportDeclaration() && elt.node.source.value === moduleSpecifier) as
+      | undefined
+      | NodePath<t.ImportDeclaration>;
+    if (declaration) {
+      let specifier = declaration
+        .get('specifiers')
+        .find(spec =>
+          exportedName === 'default'
+            ? spec.isImportDefaultSpecifier()
+            : spec.isImportSpecifier() && name(spec.node.imported) === exportedName
+        ) as undefined | NodePath<t.ImportSpecifier> | NodePath<t.ImportDefaultSpecifier>;
+      if (specifier && target.scope.getBinding(specifier.node.local.name)?.kind === 'module') {
+        return specifier.node.local;
+      } else {
+        return this.addSpecifier(target, declaration, exportedName, nameHint);
+      }
+    } else {
+      this.program.node.body.unshift(this.t.importDeclaration([], this.t.stringLiteral(moduleSpecifier)));
+      return this.addSpecifier(
+        target,
+        this.program.get(`body.0`) as NodePath<t.ImportDeclaration>,
+        exportedName,
+        nameHint
+      );
+    }
+  }
+
+  private addSpecifier(
+    target: NodePath<t.Node>,
+    declaration: NodePath<t.ImportDeclaration>,
+    exportedName: string,
+    nameHint: string | undefined
+  ): t.Identifier {
+    let local = this.t.identifier(unusedNameLike(target, desiredName(nameHint, exportedName, target)));
+    let specifier =
+      exportedName === 'default'
+        ? this.t.importDefaultSpecifier(local)
+        : this.t.importSpecifier(local, this.t.identifier(exportedName));
+    declaration.node.specifiers.push(specifier);
+    declaration.scope.registerBinding(
+      'module',
+      declaration.get(`specifiers.${declaration.node.specifiers.length - 1}`) as NodePath
+    );
+    return local;
+  }
+}
+
+function unusedNameLike(path: NodePath<unknown>, name: string): string {
+  let candidate = name;
+  let counter = 0;
+  while (path.scope.hasBinding(candidate)) {
+    candidate = `${name}${counter++}`;
+  }
+  return candidate;
+}
+
+function name(node: t.StringLiteral | t.Identifier): string {
+  if (node.type === 'StringLiteral') {
+    return node.value;
+  } else {
+    return node.name;
+  }
+}
+
+function desiredName(nameHint: string | undefined, exportedName: string, target: NodePath<t.Node>) {
+  if (nameHint) {
+    return nameHint;
+  }
+  if (exportedName === 'default') {
+    if (target.isIdentifier()) {
+      return target.node.name;
+    } else {
+      return target.scope.generateUidIdentifierBasedOnNode(target.node).name;
+    }
+  } else {
+    return exportedName;
+  }
+}