Merge remote-tracking branch 'origin/main' into wbinnssmith/ast-grep-…

…part-1

.github/workflows/nextjs-integration-test.yml

@@ -35,7 +35,7 @@ env:
   DATADOG_TRACE_NEXTJS_TEST: "true"
   DD_ENV: "ci"
   # Turbopack specific customization for the test runner
-  TURBOPACK: 1
+  EXPERIMENTAL_TURBOPACK: 1
   __INTERNAL_CUSTOM_TURBOPACK_BINDINGS: ${{ github.workspace }}/packages/next-swc/native/next-swc.linux-x64-gnu.node
   NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ github.workspace }}/integration-test-data/test-results/main/failed-test-path-list.json
   NEXT_TEST_CONTINUE_ON_ERROR: TRUE
@@ -78,8 +78,7 @@ jobs:
           npm i -g pnpm@$PNPM_VERSION && pnpm --version
           npm i -g yarn
           sudo npx playwright install-deps && pnpm playwright install
-          NEXT_TEST_MODE=dev node run-tests.js -g ${{ matrix.group }}/3 -c ${TEST_CONCURRENCY} --test-pattern '^(development|e2e|unit)/.*\.test\.(js|jsx|ts|tsx)$'
-          ls test
+          NEXT_TEST_MODE=dev node run-tests.js -g ${{ matrix.group }}/3 -c ${TEST_CONCURRENCY} --test-pattern '^(test\/(development|e2e|unit)|packages\/.*\/src\/.*)/.*\.test\.(js|jsx|ts|tsx)$'
           ls test/turbopack-test-junit-report
         # It is currently expected to fail some of next.js integration test, do not fail CI check.
         continue-on-error: true
@@ -92,7 +91,6 @@ jobs:
           name: Test trace reports
           path: |
             test/turbopack-test-junit-report
-          if-no-files-found: ignore
 
   test-prod:
     name: Next.js integration test (Production)
@@ -102,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        group: [1, 2, 3]
+        group: [1, 2, 3, 4, 5]
 
     steps:
       - uses: actions/cache/restore@v3
@@ -116,7 +114,8 @@ jobs:
           npm i -g pnpm@$PNPM_VERSION && pnpm --version
           npm i -g yarn
           sudo npx playwright install-deps && pnpm playwright install
-          NEXT_TEST_MODE=start node run-tests.js -g ${{ matrix.group }}/3 -c ${TEST_CONCURRENCY} --test-pattern '^(production|e2e)/.*\.test\.(js|jsx|ts|tsx)$'
+          NEXT_TEST_MODE=start node run-tests.js -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --test-pattern '^(test\/(production|e2e))/.*\.test\.(js|jsx|ts|tsx)$'
+          ls test/turbopack-test-junit-report
         continue-on-error: true
         env:
           NEXT_INTEGRATION_TEST: true
@@ -126,7 +125,6 @@ jobs:
           name: Test trace reports
           path: |
             test/turbopack-test-junit-report
-          if-no-files-found: ignore
 
   test-integration:
     name: Next.js integration test (Integration)
@@ -136,7 +134,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        group: [1, 2, 3, 4, 5, 6]
+        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
 
     steps:
       - uses: actions/cache/restore@v3
@@ -150,17 +148,17 @@ jobs:
           npm i -g pnpm@$PNPM_VERSION && pnpm --version
           npm i -g yarn
           sudo npx playwright install-deps && pnpm playwright install
-          node run-tests.js -g ${{ matrix.group }}/6 -c ${TEST_CONCURRENCY} --test-pattern '^(integration)/.*\.test\.(js|jsx|ts|tsx)$'
+          node run-tests.js -g ${{ matrix.group }}/12 -c ${TEST_CONCURRENCY} --test-pattern '^(test\/integration)/.*\.test\.(js|jsx|ts|tsx)$'
+          ls test/turbopack-test-junit-report
         continue-on-error: true
         env:
           NEXT_INTEGRATION_TEST: true
       - name: Upload test reports artifact
         uses: actions/upload-artifact@v3
         with:
-          name: Test trace reports
+          name: Upload test trace reports
           path: |
             test/turbopack-test-junit-report
-          if-no-files-found: ignore
 
   # Collect integration test results from execute_tests,
   # Store it as github artifact for next step to consume.
@@ -208,7 +206,7 @@ jobs:
       - name: Upload test trace to datadog
         continue-on-error: true
         run: |
-          npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci@2.14.0
+          npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci @aws-sdk/property-provider
           jrm ./nextjs-test-result-junit.xml "test/**/*.xml"
           ls -al ./*.xml
           # We'll tag this to the Turbopack service, not the next.js

.github/workflows/setup-nextjs-build.yml

@@ -88,18 +88,17 @@ jobs:
           # Basic recipe to apply patch to cargo via cli looks like this:
           # cargo check --config 'patch."https://github.com/vercel/turbo".$PKG_NAME.git="https://github.com/vercel/turbo.git?rev=$SHA"'
           # Careful to preserve quote to allow dot expression can access git url based property key.
-          export BINDING=$(printf 'patch.\\"%s\\".%s.git=\\"%s?rev=%s\\"' "$TURBOPACK_REMOTE" "turbo-binding" "$TURBOPACK_REMOTE" "$GITHUB_SHA")
+          export BINDING=$(printf 'patch.\\"%s\\".%s.git=\\"%s?rev=%s\\"' "$TURBOPACK_REMOTE" "turbopack-binding" "$TURBOPACK_REMOTE" "$GITHUB_SHA")
           export TASKS=$(printf 'patch.\\"%s\\".%s.git=\\"%s?rev=%s\\"' "$TURBOPACK_REMOTE" "turbo-tasks" "$TURBOPACK_REMOTE" "$GITHUB_SHA")
           export TASKS_FS=$(printf 'patch.\\"%s\\".%s.git=\\"%s?rev=%s\\"' "$TURBOPACK_REMOTE" "turbo-tasks-fs" "$TURBOPACK_REMOTE" "$GITHUB_SHA")
 
           echo "Trying to build next-swc with turbopack $GITHUB_SHA"
-          echo "success=false" >> $GITHUB_OUTPUT
           hyperfine --min-runs 1 --show-output 'pnpm run --filter=@next/swc build-native --features plugin,rustls-tls --release --cargo-flags="--config $BINDING --config $TASKS --config $TASKS_FS"'
-          echo "success=true" >> $GITHUB_OUTPUT
+          echo "built=pass" >> $GITHUB_OUTPUT
           echo "Successfully built next-swc with turbopack $GITHUB_SHA"
 
       - name: Build next-swc
-        if: steps.build-next-swc-turbopack-patch.outputs.success != true
+        if: steps.build-next-swc-turbopack-patch.outputs.built != 'pass'
         run: |
           echo "Looks like we could not apply latest turbopack to next-swc. Trying to build next-swc with published turbopack. This might happen when there is a breaking changes in turbopack, and next.js is not yet updated."
           hyperfine --min-runs 1 --show-output 'pnpm run --filter=@next/swc build-native --features plugin,rustls-tls --release'

.github/workflows/test.yml

@@ -1030,7 +1030,7 @@ jobs:
         env:
           DATADOG_API_KEY: ${{ secrets.DD_KEY_TURBOPACK }}
         run: |
-          npm install -g @datadog/datadog-ci
+          npm install -g @datadog/datadog-ci @aws-sdk/property-provider
           # Query raw benchmark output, create key:value pairs for each benchmark entries.
           # The generated key name is compact format the path of the benchmark entry, i.e
           # `base.hmr_to_commit.CSR.1000.mean`

Cargo.toml

@@ -45,6 +45,7 @@ default-members = [
   "crates/turbopack-swc-utils",
   "crates/turbopack-test-utils",
   "crates/turbopack-tests",
+  "crates/turbopack-wasm",
   "xtask",
 ]
 
@@ -126,6 +127,7 @@ turbopack-static = { path = "crates/turbopack-static" }
 turbopack-swc-utils = { path = "crates/turbopack-swc-utils" }
 turbopack-test-utils = { path = "crates/turbopack-test-utils" }
 turbopack-tests = { path = "crates/turbopack-tests" }
+turbopack-wasm = { path = "crates/turbopack-wasm" }
 turbopath = { path = "crates/turborepo-paths" }
 turborepo = { path = "crates/turborepo" }
 turborepo-api-client = { path = "crates/turborepo-api-client" }

crates/turbopack-ecmascript-runtime/js/src/build/runtime.ts

@@ -1,5 +1,5 @@
 /// <reference path="../shared/runtime-utils.ts" />
-/// <reference path="../shared-node/require.ts" />
+/// <reference path="../shared-node/node-utils.ts" />
 
 declare var RUNTIME_PUBLIC_PATH: string;
 
@@ -52,7 +52,7 @@ function loadChunk(chunkPath: ChunkPath) {
     return;
   }
 
-  const resolved = require.resolve(path.resolve(RUNTIME_ROOT, chunkPath));
+  const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
   const chunkModules: ModuleFactories = require(resolved);
 
   for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
@@ -74,6 +74,12 @@ function loadChunkAsync(source: SourceInfo, chunkPath: string): Promise<void> {
   });
 }
 
+function loadWebAssembly(chunkPath: ChunkPath, imports: WebAssembly.Imports) {
+  const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
+
+  return loadWebAssemblyFromPath(resolved, imports);
+}
+
 function instantiateModule(id: ModuleId, source: SourceInfo): Module {
   const moduleFactory = moduleFactories[id];
   if (typeof moduleFactory !== "function") {
@@ -134,6 +140,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
       m: module,
       c: moduleCache,
       l: loadChunkAsync.bind(null, { type: SourceType.Parent, parentId: id }),
+      w: loadWebAssembly,
       g: globalThis,
       __dirname: module.id.replace(/(^|\/)[\/]+$/, ""),
     });

crates/turbopack-ecmascript-runtime/js/src/build/tsconfig.json

@@ -1,9 +1,9 @@
 {
   "extends": "../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"],
+    // environment, we need WebWorker for WebAssembly types (not part of @types/node yet)
+    "lib": ["ESNext", "WebWorker"],
     "types": ["node"]
   },
   "include": ["*.ts"]
-}
+}

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/dummy.ts

@@ -14,3 +14,4 @@ declare var augmentContext: (
   context: TurbopackDevBaseContext
 ) => TurbopackDevContext;
 declare var commonJsRequireContext: CommonJsRequireContext;
+declare var loadWebAssembly: (source: SourceInfo, wasmChunkPath: ChunkPath, imports: WebAssembly.Imports) => Exports;

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/runtime-base.ts

@@ -316,6 +316,8 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
 
   // NOTE(alexkirsz) This can fail when the module encounters a runtime error.
   try {
+    const sourceInfo: SourceInfo = { type: SourceType.Parent, parentId: id };
+
     runModuleExecutionHooks(module, (refresh) => {
       moduleFactory.call(
         module.exports,
@@ -331,7 +333,8 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
           n: exportNamespace.bind(null, module),
           m: module,
           c: moduleCache,
-          l: loadChunk.bind(null, { type: SourceType.Parent, parentId: id }),
+          l: loadChunk.bind(null, sourceInfo),
+          w: loadWebAssembly.bind(null, sourceInfo),
           g: globalThis,
           k: refresh,
           __dirname: module.id.replace(/(^|\/)\/+$/, ""),

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/tsconfig.json

@@ -1,9 +1,8 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"],
-    "target": "ESNext"
+    // environment, we need WebWorker for WebAssembly types
+    "lib": ["ESNext", "WebWorker"]
   },
   "include": ["runtime-base.ts", "dummy.ts"]
 }

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/dom/runtime-backend-dom.ts

@@ -27,6 +27,19 @@ function commonJsRequireContext(
   return commonJsRequire(sourceModule, entry.id());
 }
 
+async function loadWebAssembly(
+  _source: SourceInfo,
+  wasmChunkPath: ChunkPath,
+  importsObj: WebAssembly.Imports
+): Promise<Exports> {
+  const chunkUrl = `/${getChunkRelativeUrl(wasmChunkPath)}`;
+
+  const req = fetch(chunkUrl);
+  const { instance } = await WebAssembly.instantiateStreaming(req, importsObj);
+
+  return instance.exports;
+}
+
 (() => {
   BACKEND = {
     async registerChunk(chunkPath, params) {

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/nodejs/runtime-backend-nodejs.ts

@@ -6,7 +6,7 @@
  */
 
 /// <reference path="../base/runtime-base.ts" />
-/// <reference path="../../../shared-node/require.ts" />
+/// <reference path="../../../shared-node/node-utils.ts" />
 
 interface RequireContextEntry {
   // Only the Node.js backend has this flag.
@@ -28,6 +28,33 @@ function augmentContext(context: TurbopackDevBaseContext): TurbopackDevContext {
   return nodejsContext;
 }
 
+function resolveChunkPath(chunkPath: ChunkPath, source: SourceInfo) {
+  let fromChunkPath = undefined;
+  switch (source.type) {
+    case SourceType.Runtime:
+      fromChunkPath = source.chunkPath;
+      break;
+    case SourceType.Parent:
+      fromChunkPath = getFirstModuleChunk(source.parentId);
+      break;
+    case SourceType.Update:
+      break;
+  }
+
+  const path = require("node:path");
+  return path.resolve(__dirname, path.posix.relative(path.dirname(fromChunkPath), chunkPath));
+}
+
+function loadWebAssembly(
+  source: SourceInfo,
+  chunkPath: ChunkPath,
+  imports: WebAssembly.Imports
+) {
+  const resolved = resolveChunkPath(chunkPath, source);
+
+  return loadWebAssemblyFromPath(resolved, imports);
+}
+
 let BACKEND: RuntimeBackend;
 
 (() => {
@@ -67,28 +94,14 @@ let BACKEND: RuntimeBackend;
       return;
     }
 
-    let fromChunkPath = undefined;
-    switch (source.type) {
-      case SourceType.Runtime:
-        fromChunkPath = source.chunkPath;
-        break;
-      case SourceType.Parent:
-        fromChunkPath = getFirstModuleChunk(source.parentId);
-        break;
-      case SourceType.Update:
-        break;
-    }
-
     // We'll only mark the chunk as loaded once the script has been executed,
     // which happens in `registerChunk`. Hence the absence of `resolve()`.
-    const path = require("path");
-    const resolved = require.resolve(
-      "./" + path.relative(path.dirname(fromChunkPath), chunkPath)
-    );
+    const resolved = resolveChunkPath(chunkPath, source);
+
     require(resolved);
   }
 })();
 
-function _eval({ code, url, map }: EcmascriptModuleEntry): ModuleFactory {
+function _eval(_: EcmascriptModuleEntry): ModuleFactory {
   throw new Error("HMR evaluation is not implemented on this backend");
 }

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/nodejs/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"],
+    // environment, we need WebWorker for WebAssembly types (not part of @types/node yet)
+    "lib": ["ESNext", "WebWorker"],
     "types": ["node"]
   },
   "include": ["*.ts"]

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/none/runtime-backend-none.ts

@@ -26,6 +26,14 @@ function commonJsRequireContext(
   return commonJsRequire(sourceModule, entry.id());
 }
 
+async function loadWebAssembly(
+  _source: SourceInfo,
+  _id: ModuleId,
+  _importsObj: any
+): Promise<Exports> {
+  throw new Error("loading WebAssemly is not supported");
+}
+
 (() => {
   BACKEND = {
     // The "none" runtime expects all chunks within the same chunk group to be

crates/turbopack-ecmascript-runtime/js/src/shared-node/require.ts → crates/turbopack-ecmascript-runtime/js/src/shared-node/node-utils.ts

@@ -46,3 +46,15 @@ externalRequire.resolve = (
 ) => {
   return require.resolve(id, options);
 };
+
+async function loadWebAssemblyFromPath(
+  path: string,
+  importsObj: WebAssembly.Imports
+): Promise<Exports> {
+  const { readFile } = require("fs/promises") as typeof import("fs/promises");
+
+  const buffer = await readFile(path);
+  const { instance } = await WebAssembly.instantiate(buffer, importsObj);
+
+  return instance.exports;
+}