Add python support for Functions Emulator. (#5423)

Add support for loading and serving functions written using the Firebase Functions Python SDK (WIP: https://github.com/firebase/firebase-functions-python)

This PR is a fork of #4653 extended with support for emulating Python functions.

Python Runtime Delegate implementation is unsurprising but does include some additional wrapper code to make sure all commands (e.g. spinning up the admin server) runs within the virtualenv environment. For now, we hardcode virtual environment `venv` directory exists on the developer's laptop, but we'll later add support for specifying arbitrary directory for specifying virtualenv directory via firebase.json configuration.

Another note is that each emulated Python function will bind to a Port instead of Unix Domain Socket (UDS) as done when emulating Node.js function. This is because there is no straightfoward, platform-neutral way to bind python webserver to UDS. Finding large number of open port might have a bit more performance penalty and cause bugs due to race condition (similar to #5418) but it seems that we have no other choice atm.

scripts/emulator-tests/functionsEmulator.spec.ts

@@ -38,6 +38,7 @@ const TEST_BACKEND = {
   secretEnv: [],
   codebase: "default",
   bin: process.execPath,
+  runtime: "nodejs14",
   // NOTE: Use the following node bin path if you want to run test cases directly from your IDE.
   // bin: path.join(MODULE_ROOT, "node_modules/.bin/ts-node"),
 };

src/deploy/functions/runtimes/discovery/index.ts

@@ -72,7 +72,7 @@ export async function detectFromPort(
 
   while (true) {
     try {
-      res = await Promise.race([fetch(`http://localhost:${port}/__/functions.yaml`), timedOut]);
+      res = await Promise.race([fetch(`http://127.0.0.1:${port}/__/functions.yaml`), timedOut]);
       break;
     } catch (err: any) {
       // Allow us to wait until the server is listening.

src/deploy/functions/runtimes/index.ts

@@ -1,6 +1,7 @@
 import * as backend from "../backend";
 import * as build from "../build";
 import * as node from "./node";
+import * as python from "./python";
 import * as validate from "../validate";
 import { FirebaseError } from "../../../error";
 
@@ -9,7 +10,7 @@ const RUNTIMES: string[] = ["nodejs10", "nodejs12", "nodejs14", "nodejs16", "nod
 // Experimental runtimes are part of the Runtime type, but are in a
 // different list to help guard against some day accidentally iterating over
 // and printing a hidden runtime to the user.
-const EXPERIMENTAL_RUNTIMES: string[] = [];
+const EXPERIMENTAL_RUNTIMES: string[] = ["python310", "python311"];
 export type Runtime = typeof RUNTIMES[number] | typeof EXPERIMENTAL_RUNTIMES[number];
 
 /** Runtimes that can be found in existing backends but not used for new functions. */
@@ -34,6 +35,8 @@ const MESSAGE_FRIENDLY_RUNTIMES: Record<Runtime | DeprecatedRuntime, string> = {
   nodejs14: "Node.js 14",
   nodejs16: "Node.js 16",
   nodejs18: "Node.js 18",
+  python310: "Python 3.10",
+  python311: "Python 3.11 (Preview)",
 };
 
 /**
@@ -113,7 +116,7 @@ export interface DelegateContext {
 }
 
 type Factory = (context: DelegateContext) => Promise<RuntimeDelegate | undefined>;
-const factories: Factory[] = [node.tryCreateDelegate];
+const factories: Factory[] = [node.tryCreateDelegate, python.tryCreateDelegate];
 
 /**
  *

src/deploy/functions/runtimes/python/index.ts

@@ -0,0 +1,152 @@
+import * as fs from "fs";
+import * as path from "path";
+import fetch from "node-fetch";
+import { promisify } from "util";
+
+import * as portfinder from "portfinder";
+
+import * as runtimes from "..";
+import * as backend from "../../backend";
+import * as discovery from "../discovery";
+import { logger } from "../../../../logger";
+import { runWithVirtualEnv } from "../../../../functions/python";
+import { FirebaseError } from "../../../../error";
+import { Build } from "../../build";
+
+const LATEST_VERSION: runtimes.Runtime = "python310";
+
+/**
+ * Create a runtime delegate for the Python runtime, if applicable.
+ *
+ * @param context runtimes.DelegateContext
+ * @return Delegate Python runtime delegate
+ */
+export async function tryCreateDelegate(
+  context: runtimes.DelegateContext
+): Promise<Delegate | undefined> {
+  const requirementsTextPath = path.join(context.sourceDir, "requirements.txt");
+
+  if (!(await promisify(fs.exists)(requirementsTextPath))) {
+    logger.debug("Customer code is not Python code.");
+    return;
+  }
+  const runtime = context.runtime ? context.runtime : LATEST_VERSION;
+  if (!runtimes.isValidRuntime(runtime)) {
+    throw new FirebaseError(`Runtime ${runtime} is not a valid Python runtime`);
+  }
+  return Promise.resolve(new Delegate(context.projectId, context.sourceDir, runtime));
+}
+
+export class Delegate implements runtimes.RuntimeDelegate {
+  public readonly name = "python";
+  constructor(
+    private readonly projectId: string,
+    private readonly sourceDir: string,
+    public readonly runtime: runtimes.Runtime
+  ) {}
+
+  private _bin = "";
+  private _modulesDir = "";
+
+  get bin(): string {
+    if (this._bin === "") {
+      this._bin = this.getPythonBinary();
+    }
+    return this._bin;
+  }
+
+  async modulesDir(): Promise<string> {
+    if (!this._modulesDir) {
+      const child = runWithVirtualEnv(
+        [
+          this.bin,
+          "-c",
+          '"import firebase_functions; import os; print(os.path.dirname(firebase_functions.__file__))"',
+        ],
+        this.sourceDir,
+        {}
+      );
+      let out = "";
+      child.stdout?.on("data", (chunk: Buffer) => {
+        const chunkString = chunk.toString();
+        out = out + chunkString;
+        logger.debug(`stdout: ${chunkString}`);
+      });
+      await new Promise((resolve, reject) => {
+        child.on("exit", resolve);
+        child.on("error", reject);
+      });
+      this._modulesDir = out.trim();
+    }
+    return this._modulesDir;
+  }
+
+  getPythonBinary(): string {
+    if (process.platform === "win32") {
+      // There is no easy way to get specific version of python executable in Windows.
+      return "python.exe";
+    }
+    if (this.runtime === "python310") {
+      return "python3.10";
+    } else if (this.runtime === "python311") {
+      return "python3.11";
+    }
+    return "python";
+  }
+
+  validate(): Promise<void> {
+    // TODO: make sure firebase-functions is included as a dep
+    return Promise.resolve();
+  }
+
+  watch(): Promise<() => Promise<void>> {
+    return Promise.resolve(() => Promise.resolve());
+  }
+
+  async build(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  async serveAdmin(port: number, envs: backend.EnvironmentVariables): Promise<() => Promise<void>> {
+    const modulesDir = await this.modulesDir();
+    const envWithAdminPort = {
+      ...envs,
+      ADMIN_PORT: port.toString(),
+    };
+    const args = [this.bin, path.join(modulesDir, "private", "serving.py")];
+    logger.debug(
+      `Running admin server with args: ${JSON.stringify(args)} and env: ${JSON.stringify(
+        envWithAdminPort
+      )} in ${this.sourceDir}`
+    );
+    const childProcess = runWithVirtualEnv(args, this.sourceDir, envWithAdminPort);
+    return Promise.resolve(async () => {
+      await fetch(`http://127.0.0.1:${port}/__/quitquitquit`);
+      const quitTimeout = setTimeout(() => {
+        if (!childProcess.killed) {
+          childProcess.kill("SIGKILL");
+        }
+      }, 10_000);
+      clearTimeout(quitTimeout);
+    });
+  }
+
+  async discoverBuild(
+    _configValues: backend.RuntimeConfigValues,
+    envs: backend.EnvironmentVariables
+  ): Promise<Build> {
+    let discovered = await discovery.detectFromYaml(this.sourceDir, this.projectId, this.runtime);
+    if (!discovered) {
+      const adminPort = await portfinder.getPortPromise({
+        port: 8081,
+      });
+      const killProcess = await this.serveAdmin(adminPort, envs);
+      try {
+        discovered = await discovery.detectFromPort(adminPort, this.projectId, this.runtime);
+      } finally {
+        await killProcess();
+      }
+    }
+    return discovered;
+  }
+}