Bump vm2 from 3.10.2 to 3.11.3

cli/vm/compile.ts

@@ -8,6 +8,7 @@ import { encode64 } from "df/common/protos";
 import { dataform } from "df/protos/ts";
 
 export function compile(compileConfig: dataform.ICompileConfig) {
+  compileConfig.projectDir = fs.realpathSync(path.resolve(compileConfig.projectDir));
   if (
     !fs.existsSync(
       path.join(compileConfig.projectDir, "node_modules", "@dataform", "core", "bundle.js")
@@ -49,7 +50,23 @@ export function compile(compileConfig: dataform.ICompileConfig) {
         path.join(parentDirName, path.relative(parentDirName, compileConfig.projectDir), moduleName)
     },
     sourceExtensions: ["js", "sql", "sqlx", "yaml", "yml"],
-    compiler
+    // vm2 3.11.3 strips file paths from V8 CallSite objects inside the sandbox,
+    // which breaks getCallerFile() in @dataform/core. Wrap each compiled module so
+    // the current file path is exposed via a global, used as a fallback when the
+    // stack-trace path is unavailable. The try/finally restores the previous value
+    // to keep nested requires (macros) consistent.
+    compiler: (code, filePath) => {
+      const compiledCode = compiler(code, filePath);
+      return `
+        var __old_file = global.__dataform_current_file;
+        global.__dataform_current_file = ${JSON.stringify(filePath)};
+        try {
+          ${compiledCode}
+        } finally {
+          global.__dataform_current_file = __old_file;
+        }
+      `;
+    }
   });
 
   const dataformCoreVersion: string = userCodeVm.run(
@@ -61,7 +78,11 @@ export function compile(compileConfig: dataform.ICompileConfig) {
   }
 
   return userCodeVm.run(
-    `return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")`,
+    `
+      global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed:", e); } })();
+      global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })();
+      return require("@dataform/core").main("${createCoreExecutionRequest(compileConfig)}")
+    `,
     vmIndexFileName
   );
 }

core/utils.ts

@@ -75,9 +75,13 @@ export function getCallerFile(rootDir: string) {
     break;
   }
   if (!lastfile) {
-    // This is likely caused by Session.compileError() being called inside Session.compile().
-    // If so, explicitly pass the filename to Session.compileError().
-    throw new Error("Unable to find valid caller file; please report this issue.");
+    if ((global as any).__dataform_current_file) {
+      lastfile = (global as any).__dataform_current_file;
+    } else {
+      // This is likely caused by Session.compileError() being called inside Session.compile().
+      // If so, explicitly pass the filename to Session.compileError().
+      throw new Error("Unable to find valid caller file; please report this issue.");
+    }
   }
   return Path.relativePath(lastfile, rootDir);
 }

core/workflow_settings.ts

@@ -10,9 +10,10 @@ declare var __non_webpack_require__: any;
 const nativeRequire = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require;
 
 export function readWorkflowSettings(failIfMissing: boolean = true): dataform.ProjectConfig {
-  const workflowSettingsYaml = maybeRequire("workflow_settings.yaml");
+  const globalAny = global as any;
+  const workflowSettingsYaml = globalAny.workflowSettingsYaml || maybeRequire("workflow_settings.yaml");
   // `dataform.json` is deprecated; new versions of Dataform Core prefer `workflow_settings.yaml`.
-  const dataformJson = maybeRequire("dataform.json");
+  const dataformJson = globalAny.dataformJson || maybeRequire("dataform.json");
 
   if (workflowSettingsYaml && dataformJson) {
     throw Error(

examples/examples_test.ts

@@ -11,7 +11,13 @@ suite("examples", { parallel: true }, () => {
 
   ["stackoverflow_reporter", "extreme_weather_programming"].forEach(exampleProject => {
     test(`${exampleProject} runs`, async () => {
-      const projectDir = `examples/${exampleProject}`;
+      // compile() calls realpath on projectDir, which would jump out of bazel's
+      // symlinked runfiles tree and leave the project without its sibling
+      // node_modules. Materialize a real copy (dereference symlinks) so the
+      // project and node_modules below resolve under a single real path.
+      const originalProjectDir = `examples/${exampleProject}`;
+      const projectDir = `examples/${exampleProject}_copy`;
+      fs.copySync(originalProjectDir, projectDir, { dereference: true });
       fs.copySync(
         "examples/node_modules/@dataform/core",
         `${projectDir}/node_modules/@dataform/core`

package.json

@@ -72,7 +72,7 @@
     "uglify-js": "^3.7.7",
     "untildify": "^4.0.0",
     "url": "^0.11.0",
-    "vm2": "3.10.2",
+    "vm2": "3.11.3",
     "vsce": "^1.79.5",
     "vscode-jsonrpc": "^5.0.1",
     "vscode-languageclient": "^6.1.3",

testing/run_core.ts

@@ -54,11 +54,12 @@ export function coreExecutionRequestFromPath(
   projectDir: string,
   projectConfigOverride?: dataform.ProjectConfig
 ): dataform.CoreExecutionRequest {
+  const resolvedProjectDir = fs.realpathSync(path.resolve(projectDir));
   return dataform.CoreExecutionRequest.create({
     compile: {
       compileConfig: {
-        projectDir,
-        filePaths: walkDirectoryForFilenames(projectDir),
+        projectDir: resolvedProjectDir,
+        filePaths: walkDirectoryForFilenames(resolvedProjectDir),
         projectConfigOverride
       }
     }
@@ -90,13 +91,28 @@ export function runMainInVm(
         path.join(parentDirName, path.relative(parentDirName, projectDir), moduleName)
     },
     sourceExtensions: SOURCE_EXTENSIONS,
-    compiler
+    compiler: (code, filePath) => {
+      const compiledCode = compiler(code, filePath);
+      return `
+        var __old_file = global.__dataform_current_file;
+        global.__dataform_current_file = ${JSON.stringify(filePath)};
+        try {
+          ${compiledCode}
+        } finally {
+          global.__dataform_current_file = __old_file;
+        }
+      `;
+    }
   });
 
   const encodedCoreExecutionRequest = encode64(dataform.CoreExecutionRequest, coreExecutionRequest);
   const vmIndexFileName = path.resolve(path.join(projectDir, "index.js"));
   const encodedCoreExecutionResponse = nodeVm.run(
-    `return require("@dataform/core").main("${encodedCoreExecutionRequest}")`,
+    `
+      global.workflowSettingsYaml = (function() { try { return require("./workflow_settings.yaml"); } catch(e) { console.error("YAML require failed run_core:", e); } })();
+      global.dataformJson = (function() { try { return require("./dataform.json"); } catch(e) {} })();
+      return require("@dataform/core").main("${encodedCoreExecutionRequest}")
+    `,
     vmIndexFileName
   );
   return decode64(dataform.CoreExecutionResponse, encodedCoreExecutionResponse);

yarn.lock

@@ -95,21 +95,6 @@
     stream-events "^1.0.5"
     teeny-request "^10.0.0"
 
-"@google-cloud/common@^3.6.0":
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b"
-  integrity "sha1-wvbaX3knmkqax8cfwC1YK+q5jos= sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q=="
-  dependencies:
-    "@google-cloud/projectify" "^2.0.0"
-    "@google-cloud/promisify" "^2.0.0"
-    arrify "^2.0.1"
-    duplexify "^4.1.1"
-    ent "^2.2.0"
-    extend "^3.0.2"
-    google-auth-library "^7.0.2"
-    retry-request "^4.1.1"
-    teeny-request "^7.0.0"
-
 "@google-cloud/common@^6.0.0":
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-6.0.0.tgz#776d4f747f0a3844f4ce13e06a2268743064b563"
@@ -155,11 +140,6 @@
   resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-4.0.0.tgz#a906e533ebdd0f754dca2509933334ce58b8c8b1"
   integrity sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==
 
-"@google-cloud/projectify@^4.0.0":
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/@google-cloud/projectify/-/projectify-4.0.0.tgz#d600e0433daf51b88c1fa95ac7f02e38e80a07be"
-  integrity sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==
-
 "@google-cloud/promisify@^4.0.0":
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-4.1.0.tgz#df8b060f0121c6462233f5420738dcda09c6df4a"
@@ -479,16 +459,6 @@
   resolved "https://registry.yarnpkg.com/@types/readline-sync/-/readline-sync-1.4.3.tgz#eac55a39d5a349912062c9e5216cd550c07fd9c8"
   integrity "sha1-6sVaOdWjSZEgYsnlIWzVUMB/2cg= sha512-YP9NVli96E+qQLAF2db+VjnAUEeZcFVg4YnMgr8kpDUFwQBnj31rPLOVHmazbKQhaIkJ9cMHsZhpKdzUeL0KTg=="
 
-"@types/request@^2.48.12":
-  version "2.48.13"
-  resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.13.tgz#abdf4256524e801ea8fdda54320f083edb5a6b80"
-  integrity sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==
-  dependencies:
-    "@types/caseless" "*"
-    "@types/node" "*"
-    "@types/tough-cookie" "*"
-    form-data "^2.5.5"
-
 "@types/request@^2.48.3":
   version "2.48.4"
   resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.4.tgz#df3d43d7b9ed3550feaa1286c6eabf0738e6cf7e"
@@ -738,7 +708,7 @@ acorn@^7.4.0:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
   integrity "sha1-/q7SVZc9LndVW4PbwIhRpsY1IPo= sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="
 
-acorn@^8.11.0, acorn@^8.14.1, acorn@^8.15.0, acorn@^8.7.1, acorn@^8.8.2:
+acorn@^8.11.0, acorn@^8.15.0, acorn@^8.7.1, acorn@^8.8.2:
   version "8.15.0"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816"
   integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==
@@ -1491,13 +1461,6 @@ domutils@^1.5.1:
     dom-serializer "0"
     domelementtype "1"
 
-dot-prop@^5.2.0:
-  version "5.2.0"
-  resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.2.0.tgz#c34ecc29556dc45f1f4c22697b6f4904e0cc4fcb"
-  integrity "sha1-w07MKVVtxF8fTCJpe29JBODMT8s= sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A=="
-  dependencies:
-    is-obj "^2.0.0"
-
 dunder-proto@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a"
@@ -1517,16 +1480,6 @@ duplexify@^4.1.3:
     readable-stream "^3.1.1"
     stream-shift "^1.0.2"
 
-duplexify@^4.1.3:
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.3.tgz#a07e1c0d0a2c001158563d32592ba58bddb0236f"
-  integrity sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==
-  dependencies:
-    end-of-stream "^1.4.1"
-    inherits "^2.0.3"
-    readable-stream "^3.1.1"
-    stream-shift "^1.0.2"
-
 ecc-jsbn@~0.1.1:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -2091,11 +2044,6 @@ get-proto@^1.0.1:
     dunder-proto "^1.0.1"
     es-object-atoms "^1.0.0"
 
-get-stream@^6.0.0:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.0.tgz#3e0012cb6827319da2706e601a1583e8629a6718"
-  integrity "sha1-PgASy2gnMZ2icG5gGhWD6GKaZxg= sha512-A1B3Bh1UmL0bidM/YX2NsCOTnGJePL9rO/M+Mw3m9f2gUpfokS0hi5Eah0WSUEWZdZhIZtMjkIYS7mDfOqNHbg=="
-
 get-value@^2.0.3, get-value@^2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
@@ -2177,6 +2125,23 @@ google-auth-library@^10.0.0-rc.1:
     gtoken "^8.0.0"
     jws "^4.0.0"
 
+google-auth-library@^9.6.3:
+  version "9.15.1"
+  resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-9.15.1.tgz#0c5d84ed1890b2375f1cd74f03ac7b806b392928"
+  integrity sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==
+  dependencies:
+    base64-js "^1.3.0"
+    ecdsa-sig-formatter "^1.0.11"
+    gaxios "^6.1.1"
+    gcp-metadata "^6.1.0"
+    gtoken "^7.0.0"
+    jws "^4.0.0"
+
+google-logging-utils@^0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/google-logging-utils/-/google-logging-utils-0.0.2.tgz#5fd837e06fa334da450433b9e3e1870c1594466a"
+  integrity sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==
+
 google-logging-utils@^1.0.0:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/google-logging-utils/-/google-logging-utils-1.1.3.tgz#17b71f1f95d266d2ddd356b8f00178433f041b17"
@@ -3965,11 +3930,6 @@ stream-shift@^1.0.2:
   resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.3.tgz#85b8fab4d71010fc3ba8772e8046cc49b8a3864b"
   integrity sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==
 
-stream-shift@^1.0.2:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.3.tgz#85b8fab4d71010fc3ba8772e8046cc49b8a3864b"
-  integrity sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==
-
 "string-width-cjs@npm:string-width@^4.2.0":
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
@@ -4002,8 +3962,7 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.1:
-  name strip-ansi-cjs
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -4017,6 +3976,13 @@ strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^7.0.1:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.2.0.tgz#d22a269522836a627af8d04b5c3fd2c7fa3e32e3"
@@ -4095,6 +4061,17 @@ teeny-request@^10.0.0:
     node-fetch "^3.3.2"
     stream-events "^1.0.5"
 
+teeny-request@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-9.0.0.tgz#18140de2eb6595771b1b02203312dfad79a4716d"
+  integrity sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==
+  dependencies:
+    http-proxy-agent "^5.0.0"
+    https-proxy-agent "^5.0.0"
+    node-fetch "^2.6.9"
+    stream-events "^1.0.5"
+    uuid "^9.0.0"
+
 terser-webpack-plugin@^5.3.16:
   version "5.3.16"
   resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz#741e448cc3f93d8026ebe4f7ef9e4afacfd56330"
@@ -4460,12 +4437,12 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
-vm2@3.10.2:
-  version "3.10.2"
-  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.10.2.tgz#b41dbe9c928b03e8a83c9b0a973f60dc7c80c5be"
-  integrity sha512-qTnbvpada8qlEEyIPFwhTcF5Ns+k83fVlOSE8XvAtHkhcQ+okMnbvryVQBfP/ExRT1CRsQpYusdATR+FBJfrnQ==
+vm2@3.11.3:
+  version "3.11.3"
+  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.11.3.tgz#5323018a93ae2862c9d52b07b658ebb8d74903f0"
+  integrity sha512-DO1TTKuOc+veL11VNOvJwRab80mghFKE40Av3bl6pdXs11bdiDMuR73owy+dS2EsTZEvRUeBkkBuDVRjV/RgEw==
   dependencies:
-    acorn "^8.14.1"
+    acorn "^8.15.0"
     acorn-walk "^8.3.4"
 
 vsce@^1.79.5: