pyodide.ts

/**
 * The main bootstrap code for loading pyodide.
 */
import {
  calculateDirname,
  loadScript,
  initNodeModules,
  resolvePath,
  loadLockFile,
  calculateInstallBaseUrl,
} from "./compat";

import { createSettings } from "./emscripten-settings";
import { version as version_ } from "./version";

import type { PyodideAPI } from "./api.js";
import type {
  TypedArray,
  PyodideModule,
  PackageData,
  FSType,
  Lockfile,
} from "./types";
import type { EmscriptenSettings } from "./emscripten-settings";
import type { SnapshotConfig } from "./snapshot";
import { withTrailingSlash } from "./common/path";
export type { PyodideAPI, TypedArray, PyodideAPI as PyodideInterface };
export type { LockfileInfo, LockfilePackage, Lockfile } from "./types";

export { type PackageData };

/**
 * The Pyodide version.
 *
 * The version here is a Python version, following :pep:`440`. This is different
 * from the version in ``package.json`` which follows the node package manager
 * version convention.
 */
export const version: string = version_;

type CreatePyodideModuleFn = (
  settings: EmscriptenSettings,
) => Promise<PyodideModule>;

// BUILD_ID is generated from hashing together pyodide.asm.mjs and
// pyodide.asm.wasm in esbuild.config.outer.mjs
//
// It is used to check that memory snapshots were generated by the same build of
// the runtime that is trying to use them. Attempting to use a snapshot from a
// different build will fail badly. See logic in snapshot.ts.
declare const BUILD_ID: string;

/**
 * The configuration options for loading Pyodide.
 */
export interface PyodideConfig {
  /**
   * The URL from which Pyodide will load the main Pyodide runtime and
   * packages. It is recommended that you leave this unchanged, providing an
   * incorrect value can cause broken behavior.
   *
   * Default: The url that Pyodide is loaded from with the file name
   * (``pyodide.js`` or ``pyodide.mjs``) removed.
   */
  indexURL?: string;
  /**
   * The file path where packages will be cached in node. If a package
   * exists in ``packageCacheDir`` it is loaded from there, otherwise it is
   * downloaded from the JsDelivr CDN and then cached into ``packageCacheDir``.
   * Only applies when running in node; ignored in browsers.
   *
   * Default: same as indexURL
   */
  packageCacheDir?: string;

  /**
   * The URL from which Pyodide will load the Pyodide ``pyodide-lock.json`` lock
   * file. You can produce custom lock files with :py:func:`micropip.freeze`.
   * Default: ```${indexURL}/pyodide-lock.json```
   */
  lockFileURL?: string;
  /**
   * The contents of a lockfile. If a string, it should be valid json and
   * ``JSON.parse()`` should return a ``Lockfile`` instance. See
   * :js:interface:`~pyodide.Lockfile` for the schema.
   */
  lockFileContents?: Lockfile | string | Promise<Lockfile | string>;
  /**
   * The base url relative to which a relative value of
   * :js:attr:`~pyodide.LockfilePackage.file_name` is interpreted. If
   * ``lockfileContents`` is provided, then ``lockFileContents`` must be
   * provided explicitly in order to install packages with relative paths.
   *
   * Otherwise, the default is calculated as follows:
   *
   * 1. If `lockFileURL` contains a ``/``, the default is everything before the last
   *    ``/`` in ``lockFileURL``.
   * 2. If in the browser, the default is ``location.toString()``.
   * 3. Otherwise, the default is `'.'`.
   */
  packageBaseUrl?: string;
  /**
   * Deprecated: This option has no effect.
   */
  fullStdLib?: boolean;
  /**
   * The URL from which to load the standard library ``python_stdlib.zip``
   * file. This URL includes the most of the Python standard library.
   * Default: ```${indexURL}/python_stdlib.zip```
   */
  stdLibURL?: string;
  /**
   * Override the standard input callback. Should ask the user for one line of
   * input. The :js:func:`pyodide.setStdin` function is more flexible and
   * should be preferred.
   */
  stdin?: () => string | null;
  /**
   * Override the standard output callback. The :js:func:`pyodide.setStdout`
   * function is more flexible and should be preferred in most cases, but
   * depending on the ``args`` passed to ``loadPyodide``, Pyodide may write to
   * stdout on startup, which can only be controlled by passing a custom
   * ``stdout`` function.
   */
  stdout?: (msg: string) => void;
  /**
   * Override the standard error output callback. The
   * :js:func:`pyodide.setStderr` function is more flexible and should be
   * preferred in most cases, but depending on the ``args`` passed to
   * ``loadPyodide``, Pyodide may write to stdout on startup, which can only
   * be controlled by passing a custom ``stdout`` function.
   */
  stderr?: (msg: string) => void;
  /**
   * The object that Pyodide will use for the ``js`` module.
   * Default: ``globalThis``
   */
  jsglobals?: object;
  /**
   * Command line arguments to pass to Python on startup. See `Python command
   * line interface options
   * <https://docs.python.org/3.10/using/cmdline.html#interface-options>`_ for
   * more details. Default: ``[]``
   */
  args?: string[];
  /**
   * Environment variables to pass to Python. This can be accessed inside of
   * Python at runtime via :py:data:`os.environ`. Certain environment variables change
   * the way that Python loads:
   * https://docs.python.org/3.10/using/cmdline.html#environment-variables
   * Default: ``{}``.
   * If ``env.HOME`` is undefined, it will be set to a default value of
   * ``"/home/pyodide"``
   */
  env?: { [key: string]: string };
  /**
   * A list of packages to load as Pyodide is initializing.
   *
   * This is the same as loading the packages with
   * :js:func:`pyodide.loadPackage` after Pyodide is loaded except using the
   * ``packages`` option is more efficient because the packages are downloaded
   * while Pyodide bootstraps itself.
   */
  packages?: string[];

  /**
   * Make loop.run_until_complete() function correctly using stack switching.
   * Default: ``true``.
   */
  enableRunUntilComplete?: boolean;

  /**
   * If true (default), throw an error if the version of Pyodide core does not
   * match the version of the Pyodide js package.
   */
  checkAPIVersion?: boolean;

  /**
   * This is a hook that allows modification of the file system before the
   * main() function is called and the intereter is started. When this is
   * called, it is guaranteed that there is an empty site-packages directory.
   * @experimental
   */
  fsInit?: (FS: FSType, info: { sitePackages: string }) => Promise<void>;

  /**
   * Opt into the old behavior where :js:func:`PyProxy.toString() <pyodide.ffi.PyProxy.toString>`
   * calls :py:func:`repr` and not :py:class:`str() <str>`. Deprecated.
   * @deprecated
   */
  pyproxyToStringRepr?: boolean;

  /**
   * Opt into the old behavior where JavaScript `null` is converted to `None`
   * instead of `jsnull`. Deprecated.
   * @deprecated
   */
  convertNullToNone?: boolean;

  /**
   * Opt into the old behavior where Python dictionaries are converted to
   * LiteralMap instead of Object.
   * @deprecated
   */
  toJsLiteralMap?: boolean;

  /**
   * Determine the value of ``sys.executable``.
   * @ignore
   */
  _sysExecutable?: string;

  /** @ignore */
  _makeSnapshot?: boolean;

  /** @ignore */
  _loadSnapshot?:
    | Uint8Array
    | ArrayBuffer
    | PromiseLike<Uint8Array | ArrayBuffer>;

  /** @ignore */
  _snapshotDeserializer?: (obj: any) => any;

  /**
   * @experimental
   * The constructor function to use to create the Pyodide module.
   * This function can be imported from `pyodide.asm.mjs`
   * and passed to `loadPyodide` as `createPyodideModule` option.
   * This is used to work around service workers forbid dynamic import(),
   * and not intended to be used in other cases.
   *
   * Warning: This is an experimental feature and may change in the future.
   */
  createPyodideModule?: CreatePyodideModuleFn;

  /** @ignore */
  BUILD_ID?: string;

  /** @ignore */
  cdnUrl?: string;
}

/**
 * @hidden
 */
export type PyodideConfigWithDefaults = Required<PyodideConfig>;

/**
 * @private
 * Initialize the configuration for Pyodide.
 * This function fills out all the fields that are not provided by the user and returns a
 */
async function initializeConfiguration(
  options: PyodideConfig = {},
): Promise<PyodideConfigWithDefaults> {
  await initNodeModules();

  if (options.lockFileContents && options.lockFileURL) {
    throw new Error("Can't pass both lockFileContents and lockFileURL");
  }

  let indexURL = options.indexURL || (await calculateDirname());
  indexURL = withTrailingSlash(resolvePath(indexURL)); // A relative indexURL causes havoc.

  options.packageBaseUrl = withTrailingSlash(options.packageBaseUrl);
  // cdnUrl only for node. withTrailingSlash is a no-op, but just in case to prevent future human errors.
  options.cdnUrl = withTrailingSlash(
    options.packageBaseUrl ??
      `https://cdn.jsdelivr.net/pyodide/v${version}/full/`,
  );

  if (!options.lockFileContents) {
    const lockFileURL = options.lockFileURL ?? indexURL + "pyodide-lock.json";
    options.lockFileContents = loadLockFile(lockFileURL);
    // packageBaseUrl isn't present, try using base location of lockFileUrl. If
    // lockFileURL is relative, use location as the base URL.
    options.packageBaseUrl ??= calculateInstallBaseUrl(lockFileURL);
  }

  options.indexURL = indexURL;

  if (options.packageCacheDir) {
    options.packageCacheDir = withTrailingSlash(
      resolvePath(options.packageCacheDir),
    );
  }

  const defaultConfig: PyodideConfig = {
    jsglobals: globalThis,
    stdin: globalThis.prompt ? () => globalThis.prompt() : undefined,
    args: [],
    env: {},
    packages: [],
    packageCacheDir: options.packageBaseUrl,
    enableRunUntilComplete: true,
    checkAPIVersion: true,
    BUILD_ID,
  };
  const config = Object.assign(
    defaultConfig,
    options,
  ) as PyodideConfigWithDefaults;
  config.env.HOME ??= "/home/pyodide";

  /**
   * `PyErr_Print()` will call `exit()` if the exception is a `SystemError`.
   * This shuts down the Python interpreter, which is a change in behavior from
   * what happened before. In order to avoid this, we set the `inspect` config
   * parameter which prevents `PyErr_Print()` from calling `exit()`. Except in
   * the cli runner, we actually do want to exit. So set default to true and in
   * cli runner we explicitly set it to false.
   */
  config.env.PYTHONINSPECT ??= "1";

  return config;
}

/**
 * @private
 */
function createEmscriptenSettings(
  config: PyodideConfigWithDefaults,
): EmscriptenSettings {
  const emscriptenSettings = createSettings(config);
  const API = emscriptenSettings.API;
  API.lockFilePromise = Promise.resolve(config.lockFileContents);

  return emscriptenSettings;
}

/**
 * @private
 */
async function loadWasmScript(
  config: PyodideConfigWithDefaults,
): Promise<CreatePyodideModuleFn> {
  if (config.createPyodideModule) {
    return config.createPyodideModule;
  }

  const scriptSrc = `${config.indexURL}pyodide.asm.mjs`;
  return (await loadScript(scriptSrc)).default;
}

/**
 * @private
 */
async function prepareSnapshot(
  config: PyodideConfigWithDefaults,
  emscriptenSettings: EmscriptenSettings,
): Promise<Uint8Array | undefined> {
  if (!config._loadSnapshot) {
    return undefined;
  }

  const snp = await config._loadSnapshot;
  const snapshot = ArrayBuffer.isView(snp)
    ? (snp as Uint8Array)
    : new Uint8Array(snp);
  emscriptenSettings.noInitialRun = true;
  // @ts-ignore
  emscriptenSettings.INITIAL_MEMORY = snapshot.length;

  return snapshot;
}

/**
 * @private
 */
async function instantiatePyodideModule(
  createPyodideModule: CreatePyodideModuleFn,
  emscriptenSettings: EmscriptenSettings,
): Promise<PyodideModule> {
  const module = await createPyodideModule(emscriptenSettings);

  // Handle early exit
  if (emscriptenSettings.exitCode !== undefined) {
    throw new module.ExitStatus(emscriptenSettings.exitCode);
  }
  return module;
}

/**
 * @private
 */
function configureAPI(
  pyodideModule: PyodideModule,
  config: PyodideConfigWithDefaults,
): void {
  const API = pyodideModule.API;

  if (config.pyproxyToStringRepr) {
    API.setPyProxyToStringMethod(true);
  }
  if (config.convertNullToNone) {
    API.setCompatNullToNone(true);
  }
  if (config.toJsLiteralMap) {
    API.setCompatToJsLiteralMap(true);
  }

  if (API.version !== version && config.checkAPIVersion) {
    throw new Error(`\
Pyodide version does not match: '${version}' <==> '${API.version}'. \
If you updated the Pyodide version, make sure you also updated the 'indexURL' parameter passed to loadPyodide.\
`);
  }

  // Disable further loading of Emscripten file_packager stuff.
  pyodideModule.locateFile = (path: string) => {
    if (path.endsWith(".so")) {
      throw new Error(`Failed to find dynamic library "${path}"`);
    }
    throw new Error(`Unexpected call to locateFile("${path}")`);
  };
}

/**
 * @private
 */
function bootstrapPyodide(
  pyodideModule: PyodideModule,
  snapshot: Uint8Array | undefined,
  config: PyodideConfigWithDefaults,
): PyodideAPI {
  const API = pyodideModule.API;

  let snapshotConfig: SnapshotConfig | undefined = undefined;
  if (snapshot) {
    snapshotConfig = API.restoreSnapshot(snapshot);
  }

  // runPython works starting after the call to finalizeBootstrap.
  const pyodide = API.finalizeBootstrap(
    snapshotConfig,
    config._snapshotDeserializer,
  );

  return pyodide;
}

/**
 * @private
 */
async function finalizeSetup(
  pyodide: PyodideAPI,
  config: PyodideConfigWithDefaults,
): Promise<PyodideAPI> {
  const API = (pyodide as any)._api;

  API.sys.path.insert(0, "");

  API._pyodide.set_excepthook();
  await API.packageIndexReady;

  // I think we want this initializeStreams call to happen after
  // packageIndexReady? I don't remember why.
  API.initializeStreams(config.stdin, config.stdout, config.stderr);

  return pyodide;
}

/**
 * Load the main Pyodide wasm module and initialize it.
 *
 * @returns The :ref:`js-api-pyodide` module.
 * @example
 * async function main() {
 *   const pyodide = await loadPyodide({
 *     stdout: (msg) => console.log(`Pyodide: ${msg}`),
 *   });
 *   console.log("Loaded Pyodide");
 * }
 * main();
 */
export async function loadPyodide(
  options: PyodideConfig = {},
): Promise<PyodideAPI> {
  // Stage 1: Initialize configuration
  const config = await initializeConfiguration(options);

  // Stage 2: Create Emscripten settings
  const emscriptenSettings = createEmscriptenSettings(config);

  // Stage 3: Load WASM script
  const createPyodideModuleFn = await loadWasmScript(config);

  // Stage 4: Prepare snapshot
  const snapshot = await prepareSnapshot(config, emscriptenSettings);

  // Stage 5: Create and initialize the Emscripten module
  const pyodideModule = await instantiatePyodideModule(
    createPyodideModuleFn,
    emscriptenSettings,
  );

  // Stage 6: Configure API and validate versions
  configureAPI(pyodideModule, config);

  // Stage 7: Bootstrap Python interpreter
  const pyodide = bootstrapPyodide(pyodideModule, snapshot, config);

  // Stage 8: Finalize setup and initialize streams
  return await finalizeSetup(pyodide, config);
}