Split off load-pyodide.js from pyodide.js (#1578)

pyodide-build/README.md

@@ -2,7 +2,8 @@
 
 Tools for building Pyodide.
 
-See [http://github.com/pyodide/pyodide](http://github.com/pyodide/pyodide) for more information.
+See [http://github.com/pyodide/pyodide](http://github.com/pyodide/pyodide) for
+more information.
 
 ## License
 

pyodide-build/pyodide_build/serve.py

@@ -5,8 +5,7 @@
 import socketserver
 import pathlib
 
-TEST_PATH = pathlib.Path(__file__).parents[0].resolve()
-BUILD_PATH = TEST_PATH / ".." / "build"
+BUILD_PATH = pathlib.Path(__file__).resolve().parents[2] / "build"
 
 
 class Handler(http.server.SimpleHTTPRequestHandler):

src/js/README.md

@@ -3,8 +3,8 @@
 The Javascript code in this folder is responsible for the following tasks:
 
 1. Defines the public Javascript API
-    - Package loading code to allow loading of other Python packages.
-    - Can load micropip.py to bootstrap loading of pure Python wheels
+   - Package loading code to allow loading of other Python packages.
+   - Can load micropip.py to bootstrap loading of pure Python wheels
 2. Loads the CPython interpreter and the core/pyodide emscripten application
    which embeds the interpreter.
 3. Injects the js/pyodide Javascript API into sys.modules. This is the final

src/js/load-pyodide.js

@@ -0,0 +1,333 @@
+import { Module } from "./module";
+
+let baseURL;
+export async function initializePackageIndex(indexURL) {
+  baseURL = indexURL;
+  let response = await fetch(`${indexURL}packages.json`);
+  Module.packages = await response.json();
+}
+
+////////////////////////////////////////////////////////////
+// Package loading
+const DEFAULT_CHANNEL = "default channel";
+
+// Regexp for validating package name and URI
+const package_uri_regexp = /^.*?([^\/]*)\.js$/;
+
+function _uri_to_package_name(package_uri) {
+  let match = package_uri_regexp.exec(package_uri);
+  if (match) {
+    return match[1];
+  }
+}
+
+export let loadScript;
+if (self.document) {
+  // browser
+  loadScript = (url) => import(url);
+} else if (self.importScripts) {
+  // webworker
+  loadScript = async (url) => {
+    // This is async only for consistency
+    self.importScripts(url);
+  };
+} else {
+  throw new Error("Cannot determine runtime environment");
+}
+
+function recursiveDependencies(
+  names,
+  _messageCallback,
+  errorCallback,
+  sharedLibsOnly
+) {
+  const packages = Module.packages.dependencies;
+  const loadedPackages = Module.loadedPackages;
+  const sharedLibraries = Module.packages.shared_library;
+  const toLoad = new Map();
+
+  const addPackage = (pkg) => {
+    if (toLoad.has(pkg)) {
+      return;
+    }
+    toLoad.set(pkg, DEFAULT_CHANNEL);
+    // If the package is already loaded, we don't add dependencies, but warn
+    // the user later. This is especially important if the loaded package is
+    // from a custom url, in which case adding dependencies is wrong.
+    if (loadedPackages[pkg] !== undefined) {
+      return;
+    }
+    for (let dep of packages[pkg]) {
+      addPackage(dep);
+    }
+  };
+  for (let name of names) {
+    const pkgname = _uri_to_package_name(name);
+    if (pkgname !== undefined) {
+      if (toLoad.has(pkgname) && toLoad.get(pkgname) !== name) {
+        errorCallback(
+          `Loading same package ${pkgname} from ${name} and ${toLoad.get(
+            pkgname
+          )}`
+        );
+        continue;
+      }
+      toLoad.set(pkgname, name);
+    } else if (name in packages) {
+      addPackage(name);
+    } else {
+      errorCallback(`Skipping unknown package '${name}'`);
+    }
+  }
+  if (sharedLibsOnly) {
+    let onlySharedLibs = new Map();
+    for (let c of toLoad) {
+      if (c[0] in sharedLibraries) {
+        onlySharedLibs.set(c[0], toLoad.get(c[0]));
+      }
+    }
+    return onlySharedLibs;
+  }
+  return toLoad;
+}
+
+async function _loadPackage(names, messageCallback, errorCallback) {
+  // toLoad is a map pkg_name => pkg_uri
+  let toLoad = recursiveDependencies(names, messageCallback, errorCallback);
+
+  // locateFile is the function used by the .js file to locate the .data
+  // file given the filename
+  Module.locateFile = (path) => {
+    // handle packages loaded from custom URLs
+    let pkg = path.replace(/\.data$/, "");
+    if (toLoad.has(pkg)) {
+      let package_uri = toLoad.get(pkg);
+      if (package_uri != DEFAULT_CHANNEL) {
+        return package_uri.replace(/\.js$/, ".data");
+      }
+    }
+    return baseURL + path;
+  };
+
+  if (toLoad.size === 0) {
+    return Promise.resolve("No new packages to load");
+  } else {
+    let packageNames = Array.from(toLoad.keys()).join(", ");
+    messageCallback(`Loading ${packageNames}`);
+  }
+
+  // This is a collection of promises that resolve when the package's JS file
+  // is loaded. The promises already handle error and never fail.
+  let scriptPromises = [];
+
+  for (let [pkg, uri] of toLoad) {
+    let loaded = Module.loadedPackages[pkg];
+    if (loaded !== undefined) {
+      // If uri is from the DEFAULT_CHANNEL, we assume it was added as a
+      // depedency, which was previously overridden.
+      if (loaded === uri || uri === DEFAULT_CHANNEL) {
+        messageCallback(`${pkg} already loaded from ${loaded}`);
+        continue;
+      } else {
+        errorCallback(
+          `URI mismatch, attempting to load package ${pkg} from ${uri} ` +
+            `while it is already loaded from ${loaded}. To override a dependency, ` +
+            `load the custom package first.`
+        );
+        continue;
+      }
+    }
+    let scriptSrc = uri === DEFAULT_CHANNEL ? `${baseURL}${pkg}.js` : uri;
+    messageCallback(`Loading ${pkg} from ${scriptSrc}`);
+    scriptPromises.push(
+      loadScript(scriptSrc).catch(() => {
+        errorCallback(`Couldn't load package from URL ${scriptSrc}`);
+        toLoad.delete(pkg);
+      })
+    );
+  }
+
+  // When the JS loads, it synchronously adds a runDependency to emscripten.
+  // It then loads the data file, and removes the runDependency from
+  // emscripten. This function returns a promise that resolves when there are
+  // no pending runDependencies.
+  function waitRunDependency() {
+    const promise = new Promise((r) => {
+      Module.monitorRunDependencies = (n) => {
+        if (n === 0) {
+          r();
+        }
+      };
+    });
+    // If there are no pending dependencies left, monitorRunDependencies will
+    // never be called. Since we can't check the number of dependencies,
+    // manually trigger a call.
+    Module.addRunDependency("dummy");
+    Module.removeRunDependency("dummy");
+    return promise;
+  }
+
+  // We must start waiting for runDependencies *after* all the JS files are
+  // loaded, since the number of runDependencies may happen to equal zero
+  // between package files loading.
+  try {
+    await Promise.all(scriptPromises).then(waitRunDependency);
+  } finally {
+    delete Module.monitorRunDependencies;
+  }
+
+  let packageList = [];
+  for (let [pkg, uri] of toLoad) {
+    Module.loadedPackages[pkg] = uri;
+    packageList.push(pkg);
+  }
+
+  let resolveMsg;
+  if (packageList.length > 0) {
+    let packageNames = packageList.join(", ");
+    resolveMsg = `Loaded ${packageNames}`;
+  } else {
+    resolveMsg = "No packages loaded";
+  }
+
+  Module.reportUndefinedSymbols();
+
+  messageCallback(resolveMsg);
+
+  // We have to invalidate Python's import caches, or it won't
+  // see the new files.
+  Module.runPythonSimple(
+    "import importlib\n" + "importlib.invalidate_caches()\n"
+  );
+}
+
+// This is a promise that is resolved iff there are no pending package loads.
+// It never fails.
+let _package_lock = Promise.resolve();
+
+/**
+ * An async lock for package loading. Prevents race conditions in loadPackage.
+ * @returns A zero argument function that releases the lock.
+ * @private
+ */
+async function acquirePackageLock() {
+  let old_lock = _package_lock;
+  let releaseLock;
+  _package_lock = new Promise((resolve) => (releaseLock = resolve));
+  await old_lock;
+  return releaseLock;
+}
+
+/**
+ *
+ * The list of packages that Pyodide has loaded.
+ * Use ``Object.keys(pyodide.loadedPackages)`` to get the list of names of
+ * loaded packages, and ``pyodide.loadedPackages[package_name]`` to access
+ * install location for a particular ``package_name``.
+ *
+ * @type {object}
+ */
+Module.loadedPackages = {};
+
+/**
+ * Load a package or a list of packages over the network. This installs the
+ * package in the virtual filesystem. The package needs to be imported from
+ * Python before it can be used.
+ * @param {String | Array | PyProxy} names Either a single package name or URL
+ * or a list of them. URLs can be absolute or relative. The URLs must have
+ * file name
+ * ``<package-name>.js`` and there must be a file called
+ * ``<package-name>.data`` in the same directory. The argument can be a
+ * ``PyProxy`` of a list, in which case the list will be converted to
+ * Javascript and the ``PyProxy`` will be destroyed.
+ * @param {function} messageCallback A callback, called with progress messages
+ *    (optional)
+ * @param {function} errorCallback A callback, called with error/warning
+ *    messages (optional)
+ * @async
+ */
+Module.loadPackage = async function (names, messageCallback, errorCallback) {
+  if (Module.isPyProxy(names)) {
+    let temp;
+    try {
+      temp = names.toJs();
+    } finally {
+      names.destroy();
+    }
+    names = temp;
+  }
+
+  if (!Array.isArray(names)) {
+    names = [names];
+  }
+  // get shared library packages and load those first
+  // otherwise bad things happen with linking them in firefox.
+  let sharedLibraryNames = [];
+  try {
+    let sharedLibraryPackagesToLoad = recursiveDependencies(
+      names,
+      messageCallback,
+      errorCallback,
+      true
+    );
+    for (let pkg of sharedLibraryPackagesToLoad) {
+      sharedLibraryNames.push(pkg[0]);
+    }
+  } catch (e) {
+    // do nothing - let the main load throw any errors
+  }
+  // override the load plugin so that it imports any dlls also
+  // this only needs to be done for shared library packages because
+  // we assume that if a package depends on a shared library
+  // it needs to have access to it.
+  // not needed for so in standard module because those are linked together
+  // correctly, it is only where linking goes across modules that it needs to
+  // be done. Hence we only put this extra preload plugin in during the shared
+  // library load
+  let oldPlugin;
+  for (let p in Module.preloadPlugins) {
+    if (Module.preloadPlugins[p].canHandle("test.so")) {
+      oldPlugin = Module.preloadPlugins[p];
+      break;
+    }
+  }
+  let dynamicLoadHandler = {
+    get: function (obj, prop) {
+      if (prop === "handle") {
+        return function (bytes, name) {
+          obj[prop].apply(obj, arguments);
+          this["asyncWasmLoadPromise"] = this["asyncWasmLoadPromise"].then(
+            function () {
+              Module.loadDynamicLibrary(name, {
+                global: true,
+                nodelete: true,
+              });
+            }
+          );
+        };
+      } else {
+        return obj[prop];
+      }
+    },
+  };
+  var loadPluginOverride = new Proxy(oldPlugin, dynamicLoadHandler);
+  // restore the preload plugin
+  Module.preloadPlugins.unshift(loadPluginOverride);
+
+  let releaseLock = await acquirePackageLock();
+  try {
+    await _loadPackage(
+      sharedLibraryNames,
+      messageCallback || console.log,
+      errorCallback || console.error
+    );
+    Module.preloadPlugins.shift(loadPluginOverride);
+    await _loadPackage(
+      names,
+      messageCallback || console.log,
+      errorCallback || console.error
+    );
+  } finally {
+    releaseLock();
+  }
+};

src/js/module.js

@@ -0,0 +1,5 @@
+export let Module = {};
+Module.noImageDecoding = true;
+Module.noAudioDecoding = true;
+Module.noWasmDecoding = false; // we preload wasm using the built in plugin now
+Module.preloadedWasm = {};

src/js/package.json

@@ -1,5 +1,5 @@
 {
-  "name" : "pyodide",
+  "name": "pyodide",
   "version": "0.18.0dev0",
   "dependencies": {
     "prettier": "^2.2.1",