Node's 'vm' not secure

[bradvogel]

Node's vm module isn't secure, so if an LLM is tricked into writing malicious code, it could be jailbroken.

Perhaps consider https://github.com/laverdet/isolated-vm instead?

[h3xxit]

Yep looking into it!

[h3xxit]

@universal-tool-calling-protocol/contributors would love some support on this issue. Anyone that has some time is free to take it.

[edujuan]

How do you see the implementation? Using isolated-vm?

[bradvogel]

Here's some reference code to get you started. I wrote it for a closed source project, though happy to share here.

runtime.ts

import dedent from 'dedent';
import * as esbuild from 'esbuild';

import { formatTransformFailure, isTransformSyntaxError, serializeRuntimeError } from './error-utils';

// Require form is necessary because isolated-vm uses a prototype-based export that doesn't expose its members as own properties.
const ivm: typeof import('isolated-vm') = require('isolated-vm');

const DEFAULT_MEMORY_LIMIT_MB = 128;
const DEFAULT_TIMEOUT_MS = 30_000;

export interface ExecuteIsolateOptions {
  code: string;
  signal: AbortSignal;
  timeout?: number; // Tests use this
}

export interface ExecuteIsolateResult {
  logs: CapturedLogEntry[];
}

export interface CapturedLogEntry {
  level: string;
  message: unknown[];
}

const wrapUserProgram = (code: string): string =>
  [
    '(async () => {', 
     code,
     '})()',
     '  .catch((error) => {',
     '    throw error;', '  });'
 ].join('\n');

const throwIfAborted = (signal: AbortSignal): void => {
  if (!signal.aborted) {
    return;
  }

  const reason = (signal as AbortSignal & { reason?: unknown }).reason;
  if (reason instanceof Error) {
    throw reason;
  }

  const error = new Error(typeof reason === 'string' ? reason : 'Execution aborted');
  error.name = 'AbortError';
  throw error;
};

export async function executeCodeInIsolate({
  code,
  signal,
  timeout = DEFAULT_TIMEOUT_MS,
}: ExecuteIsolateOptions): Promise<ExecuteIsolateResult> {
  throwIfAborted(signal);

  const transformedCode = `wrapUserProgram`(code);

  let jsCode: string;
  try {
    const transformed = await esbuild.transform(transformedCode, {
      loader: 'ts',
      format: 'cjs',
      jsx: 'transform',
      platform: 'neutral',
      target: 'es2020',
      sourcemap: false
    });
    jsCode = transformed.code;
  } catch (error: unknown) {
    // Syntax errors get special formatting for the LLM.
    if (isTransformSyntaxError(error)) {
      return {
        logs: [
          {
            level: 'fatal',
            message: [formatTransformFailure(error)]
          }
        ]
      };
    }

    // Unknown errors should be rethrown.
    throw error;
  }

  throwIfAborted(signal);

  const isolate = new ivm.Isolate({ memoryLimit: DEFAULT_MEMORY_LIMIT_MB });
  const context = await isolate.createContext();

  const logs: CapturedLogEntry[] = [];

  await context.evalClosure(
    dedent(`
      globalThis.console = {
        log: (...args) => $0('log', ...args),
        info: (...args) => $0('info', ...args),
        warn: (...args) => $0('warn', ...args),
        error: (...args) => $0('error', ...args)
      };
    `),
    [
      (level: string, ...messages: unknown[]) => {
        logs.push({ level, message: messages });
      }
    ]
  );

  const abortHandler = () => {
    try {
      isolate.dispose();
    } catch {
      // Ignore dispose errors during abort.
    }
  };

  signal.addEventListener('abort', abortHandler);

  try {
    const script = await isolate.compileScript(jsCode, {
      filename: 'execute-code',
      lineOffset: -1 // Ignore wrapper start
    });
    await script.run(context, {
      timeout
    });

    return {
      logs
    };
  } catch (error) {
    logs.push({ level: 'fatal', message: [serializeRuntimeError(error)] });
    return {
      logs
    };
  } finally {
    signal.removeEventListener('abort', abortHandler);
    isolate.dispose();
  }
}

runtime.test.ts

import dedent from 'dedent';
import { executeCodeInIsolate } from './index';

afterEach(() => {
  jest.restoreAllMocks();
});

describe('executeCodeInIsolate', () => {
  describe('logging', () => {
    it('records console output for successful execution', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          const sum = [1, 2, 3].reduce((acc, value) => acc + value, 0);
          console.log('sum', sum);
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "log",
              "message": [
                "sum",
                6,
              ],
            },
          ],
        }
      `);
    });

    it('logs Date strings when requested', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          const ts = new Date('2025-05-15T12:34:56Z').toString();
          console.log(ts);
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "log",
              "message": [
                "Thu May 15 2025 08:34:56 GMT-0400 (Eastern Daylight Time)",
              ],
            },
          ],
        }
      `);
    });

    it('logs plain objects', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.log({ alpha: 1, nested: { beta: true } });
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "log",
              "message": [
                {
                  "alpha": 1,
                  "nested": {
                    "beta": true,
                  },
                },
              ],
            },
          ],
        }
      `);
    });

    it('logs arrays with mixed entries', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.log([1, 'two', { three: 3 }]);
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "log",
              "message": [
                [
                  1,
                  "two",
                  {
                    "three": 3,
                  },
                ],
              ],
            },
          ],
        }
      `);
    });

    it('logs multiple arguments with console.log', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.log('alpha', 42, { nested: true });
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "log",
              "message": [
                "alpha",
                42,
                {
                  "nested": true,
                },
              ],
            },
          ],
        }
      `);
    });

    it('logs multiple arguments with console.info', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.info('beta', false, [1, 2, 3]);
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "info",
              "message": [
                "beta",
                false,
                [
                  1,
                  2,
                  3,
                ],
              ],
            },
          ],
        }
      `);
    });

    it('logs multiple arguments with console.warn', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.warn('gamma', { warning: true }, null);
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "warn",
              "message": [
                "gamma",
                {
                  "warning": true,
                },
                null,
              ],
            },
          ],
        }
      `);
    });

    it('logs Error instances', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.error(new Error('sad times'));
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "error",
              "message": [
                [Error: sad times],
              ],
            },
          ],
        }
      `);
    });

    it('logs multiple arguments with console.error', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.error('delta', new Error('bad'), { code: 500 });
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "error",
              "message": [
                "delta",
                [Error: bad],
                {
                  "code": 500,
                },
              ],
            },
          ],
        }
      `);
    });

    it('captures console output across common levels', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.log('alpha');
          console.info('beta');
          console.warn('gamma');
          console.error('delta');
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "log",
              "message": [
                "alpha",
              ],
            },
            {
              "level": "info",
              "message": [
                "beta",
              ],
            },
            {
              "level": "warn",
              "message": [
                "gamma",
              ],
            },
            {
              "level": "error",
              "message": [
                "delta",
              ],
            },
          ],
        }
      `);
    });
  });

  describe('runtime errors', () => {
    it('surfaces thrown errors back to the caller', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          console.error('something failed');
          throw new Error('boom');
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "error",
              "message": [
                "something failed",
              ],
            },
            {
              "level": "fatal",
              "message": [
                "Error: boom
            at execute-code:2:9
            at execute-code:3:3
            at (<isolated-vm boundary>)
              ],
            },
          ],
        }
      `);
    });

    it('surfaces custom error classes thrown within the isolate (as Error)', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          class CustomIsolateError extends Error {
            constructor(message) {
              super(message);
              this.name = 'CustomIsolateError';
            }
          }

          throw new CustomIsolateError('kaboom');
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "Error [CustomIsolateError]: kaboom
            at execute-code:7:9
            at execute-code:8:3
            at (<isolated-vm boundary>)
              ],
            },
          ],
        }
      `);
    });

    it('times out if the snippet never completes', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          while (true) {}
        `,
        signal: controller.signal,
        timeout: 1_000
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "Error: Script execution timed out.
            at execute-code:1:3
            at (<isolated-vm boundary>)
              ],
            },
          ],
        }
      `);
    });

    it('surfaces literal string throws verbatim', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          throw 'literal boom';
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "RuntimeError: 'literal boom'",
              ],
            },
          ],
        }
      `);
    });

    it('describes undefined throws explicitly', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          throw undefined;
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "RuntimeError: undefined",
              ],
            },
          ],
        }
      `);
    });

    it('describes null throws explicitly', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          throw null;
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "RuntimeError: null",
              ],
            },
          ],
        }
      `);
    });

    it('includes stack traces in runtime error details', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          function bar() {
            throw new Error('boom');
          }

          function foo() {
            bar();
          }

          foo();
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "Error: boom
            at bar (execute-code:2:11)
            at foo (execute-code:5:5)
            at execute-code:7:3
            at execute-code:8:3
            at (<isolated-vm boundary>)
              ],
            },
          ],
        }
      `);
    });

    it('surfaces structured non-Error throws with descriptive messages', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
        const err = { reason: 'nah', meta: { attempt: 2 } };
        throw err;
      `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "Error: An object was thrown from supplied code within isolated-vm, but that object was not an instance of \`Error\`.
            at (<isolated-vm boundary>)
              ],
            },
          ],
        }
      `);
    });

    it('stringifies primitive throws', async () => {
      const cases = [
        { literal: '42', expected: 'RuntimeError: 42' },
        { literal: 'false', expected: 'RuntimeError: false' },
        { literal: '123n', expected: 'RuntimeError: 123n' }
      ];

      for (const { literal, expected } of cases) {
        const controller = new AbortController();

        const output = await executeCodeInIsolate({
          code: dedent`
            const value = ${literal};
            throw value;
          `,
          signal: controller.signal
        });

        expect(output.logs.at(-1)).toEqual({ level: 'fatal', message: [expected] });
      }
    });

    it('omits stack traces even for custom errors', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          const err = new Error('danger');
          err.name = 'CustomBoom';
          err.stack = 'fake-stack';
          err.meta = { reason: 'custom', attempt: 3 };
          throw err;
        `,
        signal: controller.signal
      });

      expect(output).toMatchInlineSnapshot(`
        {
          "logs": [
            {
              "level": "fatal",
              "message": [
                "Error: danger
            at (<isolated-vm boundary>)
              ],
            },
          ],
        }
      `);
    });
  });

  describe('transform errors', () => {
    it('handles compilation errors gracefully', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          this is not valid javascript code;
        `,
        signal: controller.signal
      });

      expect(output).toEqual({
        logs: [
          {
            level: 'fatal',
            message: [
              [
                'this is not valid javascript code;',
                '     ^^',
                '',
                'Uncaught SyntaxError[1:5]: Expected ";" but found "is"'
              ].join('\n')
            ]
          }
        ]
      });
    });

    it('reports accurate locations for later parse errors', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          const ok = 1;
          function noop() {
            return ok;
          }

          const nope = ;
        `,
        signal: controller.signal
      });

      expect(output).toEqual({
        logs: [
          {
            level: 'fatal',
            message: [
              [
                'const nope = ;',
                '             ^',
                '',
                'Uncaught SyntaxError[6:13]: Unexpected ";"'
              ].join('\n')
            ]
          }
        ]
      });
    });

    it('handles missing function name parse errors', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          function() {}
        `,
        signal: controller.signal
      });

      expect(output).toEqual({
        logs: [
          {
            level: 'fatal',
            message: [
              [
                'function() {}',
                '        ^',
                '',
                'Uncaught SyntaxError[1:8]: Expected identifier but found "("'
              ].join('\n')
            ]
          }
        ]
      });
    });

    it('reports only the first parse error (because this is all esbuild gives us)', async () => {
      const controller = new AbortController();

      const output = await executeCodeInIsolate({
        code: dedent`
          function foo() {
            if (true {
              console.log('oops');
            }

            const x = ;   // also invalid
          }
        `,
        signal: controller.signal
      });

      // Note that esbuild currently stops after the first parse error, so we assert that the
      // surfaced location matches the invalid `if` condition.
      expect(output).toEqual({
        logs: [
          {
            level: 'fatal',
            message: [
              [
                'if (true {',
                '         ^',
                '',
                'Uncaught SyntaxError[2:9]: Expected ")" but found "{"'
              ].join('\n')
            ]
          }
        ]
      });
    });
  });

  it('respects the abort signal', async () => {
    const controller = new AbortController();

    const promise = executeCodeInIsolate({
      code: dedent`
        while (true) {}
      `,
      signal: controller.signal
    });

    // Abort immediately
    controller.abort();

    await expect(promise).rejects.toThrow('Execution aborted');
  });
});

error-utils.ts is specific to my implementation, so I didn't include that.